Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit e1ba9cde authored by Varun Shah's avatar Varun Shah
Browse files

Obfuscate usage stats data stored on disk. 

All of the usage stats data stored on disk will now be obfuscated. There
will be a package to tokens mappings file stored on disk which has a
hierarchy of mappings for each string in each package's usage stats
data. A UsageStatsProtoV2 was added to keep the logic clean and separate
from the original usage stats proto parser.

Initial observations show a memory gain of over 60% w.r.t. the usage
stats data size on disk. There is also no performance hit because of this
change - in fact, even with the obfuscation overhead, reads are now over
65% faster and writes are up to 50% faster.

Bug: 135484470
Test: atest UsageStatsTest
Test: atest UsageStatsDatabaseTest
Test: atest UsageStatsDatabasePerfTest
Change-Id: I55ce729033d8b6e4051271802d57c72684053c32
parent 76e56e40

core/java/android/app/usage/UsageEvents.java

+35 −0
Original line number Diff line number Diff line
@@ -311,18 +311,33 @@ public final class UsageEvents implements Parcelable { 
         */

        public static final int VALID_FLAG_BITS = FLAG_IS_PACKAGE_INSTANT_APP;



        /**

         * @hide

         */

        private static final int UNASSIGNED_TOKEN = -1;



        /**

         * {@hide}

         */

        @UnsupportedAppUsage(maxTargetSdk = Build.VERSION_CODES.P, trackingBug = 115609023)

        public String mPackage;



        /**

         * {@hide}

         */

        public int mPackageToken = UNASSIGNED_TOKEN;



        /**

         * {@hide}

         */

        @UnsupportedAppUsage(maxTargetSdk = Build.VERSION_CODES.P, trackingBug = 115609023)

        public String mClass;



        /**

         * {@hide}

         */

        public int mClassToken = UNASSIGNED_TOKEN;



        /**

         * {@hide}

         */
@@ -333,11 +348,21 @@ public final class UsageEvents implements Parcelable { 
         */

        public String mTaskRootPackage;



        /**

         * {@hide}

         */

        public int mTaskRootPackageToken = UNASSIGNED_TOKEN;



        /**

         * {@hide}

         */

        public String mTaskRootClass;



        /**

         * {@hide}

         */

        public int mTaskRootClassToken = UNASSIGNED_TOKEN;



        /**

         * {@hide}

         */
@@ -364,6 +389,11 @@ public final class UsageEvents implements Parcelable { 
         */

        public String mShortcutId;



        /**

         * {@hide}

         */

        public int mShortcutIdToken = UNASSIGNED_TOKEN;



        /**

         * Action type passed to ChooserActivity

         * Only present for {@link #CHOOSER_ACTION} event types.
@@ -401,6 +431,11 @@ public final class UsageEvents implements Parcelable { 
         */

        public String mNotificationChannelId;



        /**

         * {@hide}

         */

        public int mNotificationChannelIdToken = UNASSIGNED_TOKEN;



        /** @hide */

        @EventFlags

        public int mFlags;

core/java/android/app/usage/UsageStats.java

+11 −0
Original line number Diff line number Diff line
@@ -35,6 +35,7 @@ import android.os.Bundle; 
import android.os.Parcel;

import android.os.Parcelable;

import android.util.ArrayMap;

import android.util.SparseArray;

import android.util.SparseIntArray;



/**
@@ -49,6 +50,11 @@ public final class UsageStats implements Parcelable { 
    @UnsupportedAppUsage(maxTargetSdk = Build.VERSION_CODES.P, trackingBug = 115609023)

    public String mPackageName;



    /**

     * {@hide}

     */

    public int mPackageToken = -1;



    /**

     * {@hide}

     */
@@ -140,6 +146,11 @@ public final class UsageStats implements Parcelable { 
     */

    public ArrayMap<String, ArrayMap<String, Integer>> mChooserCounts = new ArrayMap<>();



    /**

     * {@hide}

     */

    public SparseArray<SparseIntArray> mChooserCountsObfuscated = new SparseArray<>();



    /**

     * {@hide}

     */

core/proto/android/server/usagestatsservice.proto

+0 −2
Original line number Diff line number Diff line
@@ -114,6 +114,4 @@ message IntervalStatsProto { 
  repeated UsageStats packages = 20;

  repeated Configuration configurations = 21;

  repeated Event event_log = 22;



  repeated Event pending_events = 23; // TODO: move to usagestatsservice_v2.proto

}

core/proto/android/server/usagestatsservice_v2.proto

0 → 100644
+135 −0
Original line number Diff line number Diff line 
/*

 * Copyright (C) 2019 The Android Open Source Project

 *

 * Licensed under the Apache License, Version 2.0 (the "License");

 * you may not use this file except in compliance with the License.

 * You may obtain a copy of the License at

 *

 *      http://www.apache.org/licenses/LICENSE-2.0

 *

 * Unless required by applicable law or agreed to in writing, software

 * distributed under the License is distributed on an "AS IS" BASIS,

 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

 * See the License for the specific language governing permissions and

 * limitations under the License.

 */



syntax = "proto2";

package com.android.server.usage;

import "frameworks/base/core/proto/android/content/configuration.proto";

import "frameworks/base/core/proto/android/privacy.proto";



option java_multiple_files = true;



/**

 * Obfuscated version of android.service.IntervalStatsProto (usagestatsservice.proto).

 */

message IntervalStatsObfuscatedProto {



  message CountAndTime {

    optional int32 count = 1;

    optional int64 time_ms = 2;

  }



  // Stores the relevant information an IntervalStats will have about a Configuration

  message Configuration {

    optional .android.content.ConfigurationProto config = 1;

    optional int64 last_time_active_ms = 2;

    optional int64 total_time_active_ms = 3;

    optional int32 count = 4;

    optional bool active = 5;

  }



  // The following fields contain supplemental data used to build IntervalStats.

  optional int64 end_time_ms = 1;

  optional int32 major_version = 2;

  optional int32 minor_version = 3;



  // The following fields contain aggregated usage stats data

  optional CountAndTime interactive = 10;

  optional CountAndTime non_interactive = 11;

  optional CountAndTime keyguard_shown = 12;

  optional CountAndTime keyguard_hidden = 13;



  // The following fields contain listed usage stats data

  repeated UsageStatsObfuscatedProto packages = 20;

  repeated Configuration configurations = 21;

  repeated EventObfuscatedProto event_log = 22;

  // The following field is only used to persist the reported events before a user unlock

  repeated PendingEventProto pending_events = 23;

}



/**

 * Stores the relevant information from an obfuscated UsageStats.

 */

message UsageStatsObfuscatedProto {

  message ChooserAction {

    message CategoryCount {

      optional int32 category_token = 1;

      optional int32 count = 2;

    }

    optional int32 action_token = 1;

    repeated CategoryCount counts = 2;

  }

  optional int32 package_token = 1;

  optional int64 last_time_active_ms = 3;

  optional int64 total_time_active_ms = 4;

  optional int32 last_event = 5;

  optional int32 app_launch_count = 6;

  repeated ChooserAction chooser_actions = 7;

  optional int64 last_time_service_used_ms = 8;

  optional int64 total_time_service_used_ms = 9;

  optional int64 last_time_visible_ms = 10;

  optional int64 total_time_visible_ms = 11;

}



/**

 * Stores the relevant information from an obfuscated Event.

 */

message EventObfuscatedProto {

  optional int32 package_token = 1;

  optional int32 class_token = 2;

  optional int64 time_ms = 3;

  optional int32 flags = 4;

  optional int32 type = 5;

  optional .android.content.ConfigurationProto config = 6;

  optional int32 shortcut_id_token = 7;

  optional int32 standby_bucket = 8;

  optional int32 notification_channel_id_token = 9;

  optional int32 instance_id = 10;

  optional int32 task_root_package_token = 11;

  optional int32 task_root_class_token = 12;

}



/**

 * This message stores all of the fields in an Event object as strings instead of tokens.

 */

message PendingEventProto {

  optional string package_name = 1;

  optional string class_name = 2;

  optional int64 time_ms = 3;

  optional int32 flags = 4;

  optional int32 type = 5;

  optional .android.content.ConfigurationProto config = 6;

  optional string shortcut_id = 7;

  optional int32 standby_bucket = 8;

  optional string notification_channel_id = 9;

  optional int32 instance_id = 10;

  optional string task_root_package = 11;

  optional string task_root_class = 12;

}



/**

 * A proto message representing the obfuscated tokens mappings for Usage Stats.

 */

message ObfuscatedPackagesProto {

  message PackagesMap {

    optional int32 package_token = 1;

    // The list of strings for each package where their indices are the token

    repeated string strings = 2;

  }



  optional int32 counter = 1;

  // Stores the mappings for every package

  repeated PackagesMap packages_map = 2;

}

services/tests/servicestests/src/com/android/server/usage/UsageStatsDatabaseTest.java

+70 −1
Original line number Diff line number Diff line
@@ -48,7 +48,7 @@ import java.util.Locale; 
@SmallTest

public class UsageStatsDatabaseTest {



    private static final int MAX_TESTED_VERSION = 4;

    private static final int MAX_TESTED_VERSION = 5;

    protected Context mContext;

    private UsageStatsDatabase mUsageStatsDatabase;

    private File mTestDir;
@@ -259,6 +259,24 @@ public class UsageStatsDatabaseTest { 


    void compareUsageEvent(Event e1, Event e2, int debugId, int minVersion) {

        switch (minVersion) {

            case 5: // test fields added in version 5

                assertEquals(e1.mPackageToken, e2.mPackageToken, "Usage event " + debugId);

                assertEquals(e1.mClassToken, e2.mClassToken, "Usage event " + debugId);

                assertEquals(e1.mTaskRootPackageToken, e2.mTaskRootPackageToken,

                        "Usage event " + debugId);

                assertEquals(e1.mTaskRootClassToken, e2.mTaskRootClassToken,

                        "Usage event " + debugId);

                switch (e1.mEventType) {

                    case Event.SHORTCUT_INVOCATION:

                        assertEquals(e1.mShortcutIdToken, e2.mShortcutIdToken,

                                "Usage event " + debugId);

                        break;

                    case Event.NOTIFICATION_INTERRUPTION:

                        assertEquals(e1.mNotificationChannelIdToken, e2.mNotificationChannelIdToken,

                                "Usage event " + debugId);

                        break;

                }

                // fallthrough

            case 4: // test fields added in version 4

                assertEquals(e1.mInstanceId, e2.mInstanceId, "Usage event " + debugId);

                assertEquals(e1.mTaskRootPackage, e2.mTaskRootPackage, "Usage event " + debugId);
@@ -372,6 +390,9 @@ public class UsageStatsDatabaseTest { 
        UsageStatsDatabase prevDB = new UsageStatsDatabase(mTestDir, oldVersion);

        prevDB.init(1);

        prevDB.putUsageStats(interval, mIntervalStats);

        if (oldVersion >= 5) {

            prevDB.writeMappingsLocked();

        }



        // Simulate an upgrade to a new version and read from the disk

        UsageStatsDatabase newDB = new UsageStatsDatabase(mTestDir, newVersion);
@@ -438,6 +459,28 @@ public class UsageStatsDatabaseTest { 
        runVersionChangeTest(3, 4, UsageStatsManager.INTERVAL_YEARLY);

    }



    /**

     * Test the version upgrade from 4 to 5

     */

    @Test

    public void testVersionUpgradeFrom4to5() throws IOException {

        runVersionChangeTest(4, 5, UsageStatsManager.INTERVAL_DAILY);

        runVersionChangeTest(4, 5, UsageStatsManager.INTERVAL_WEEKLY);

        runVersionChangeTest(4, 5, UsageStatsManager.INTERVAL_MONTHLY);

        runVersionChangeTest(4, 5, UsageStatsManager.INTERVAL_YEARLY);

    }



    /**

     * Test the version upgrade from 3 to 5

     */

    @Test

    public void testVersionUpgradeFrom3to5() throws IOException {

        runVersionChangeTest(3, 5, UsageStatsManager.INTERVAL_DAILY);

        runVersionChangeTest(3, 5, UsageStatsManager.INTERVAL_WEEKLY);

        runVersionChangeTest(3, 5, UsageStatsManager.INTERVAL_MONTHLY);

        runVersionChangeTest(3, 5, UsageStatsManager.INTERVAL_YEARLY);

    }





    /**

     * Test the version upgrade from 3 to 4
@@ -492,4 +535,30 @@ public class UsageStatsDatabaseTest { 
            assertEquals(extra, files.keyAt(0));

        }

    }



    private void compareObfuscatedData(int interval) throws IOException {

        // Write IntervalStats to disk

        UsageStatsDatabase prevDB = new UsageStatsDatabase(mTestDir, 5);

        prevDB.init(1);

        prevDB.putUsageStats(interval, mIntervalStats);

        prevDB.writeMappingsLocked();



        // Read IntervalStats from disk into a new db

        UsageStatsDatabase newDB = new UsageStatsDatabase(mTestDir, 5);

        newDB.init(mEndTime);

        List<IntervalStats> stats = newDB.queryUsageStats(interval, 0, mEndTime,

                mIntervalStatsVerifier);



        assertEquals(1, stats.size());

        // The written and read IntervalStats should match

        compareIntervalStats(mIntervalStats, stats.get(0), 5);

    }



    @Test

    public void testObfuscation() throws IOException {

        compareObfuscatedData(UsageStatsManager.INTERVAL_DAILY);

        compareObfuscatedData(UsageStatsManager.INTERVAL_WEEKLY);

        compareObfuscatedData(UsageStatsManager.INTERVAL_MONTHLY);

        compareObfuscatedData(UsageStatsManager.INTERVAL_YEARLY);

    }

}