Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 103ebf5b authored by Ricky Wai's avatar Ricky Wai
Browse files

Apply differential privacy on watchlist report 

Changes:
- Added watchlist report generation functions
- Apply DP to watchlist report
- Renamed WatchlistSettings as WatchlistConfig
- WatchlistSettings now stores user generated settings

Bug: 63908748
Test: Able to compile, tests pass

Change-Id: I5a4e3f7e5e1195203aa0a66fa4b9e7ed4e0c1b27
parent 1f31d9a8

services/core/java/com/android/server/net/watchlist/NetworkWatchlistService.java

+5 −11
Original line number Diff line number Diff line
@@ -17,15 +17,11 @@ 
package com.android.server.net.watchlist;



import android.content.Context;

import android.content.pm.PackageManager;

import android.content.pm.PackageManager.NameNotFoundException;

import android.net.IIpConnectivityMetrics;

import android.net.INetdEventCallback;

import android.net.NetworkWatchlistManager;

import android.net.metrics.IpConnectivityLog;

import android.os.Binder;

import android.os.Process;

import android.os.SharedMemory;

import android.os.RemoteException;

import android.os.ServiceManager;

import android.os.SystemProperties;
@@ -42,9 +38,7 @@ import com.android.server.ServiceThread; 
import com.android.server.SystemService;



import java.io.FileDescriptor;

import java.io.IOException;

import java.io.PrintWriter;

import java.util.List;



/**

 * Implementation of network watchlist service.
@@ -99,7 +93,7 @@ public class NetworkWatchlistService extends INetworkWatchlistManager.Stub { 
    private volatile boolean mIsLoggingEnabled = false;

    private final Object mLoggingSwitchLock = new Object();



    private final WatchlistSettings mSettings;

    private final WatchlistConfig mConfig;

    private final Context mContext;



    // Separate thread to handle expensive watchlist logging work.
@@ -112,7 +106,7 @@ public class NetworkWatchlistService extends INetworkWatchlistManager.Stub { 


    public NetworkWatchlistService(Context context) {

        mContext = context;

        mSettings = WatchlistSettings.getInstance();

        mConfig = WatchlistConfig.getInstance();

        mHandlerThread = new ServiceThread(TAG, Process.THREAD_PRIORITY_BACKGROUND,

                        /* allowIo */ false);

        mHandlerThread.start();
@@ -126,7 +120,7 @@ public class NetworkWatchlistService extends INetworkWatchlistManager.Stub { 
    NetworkWatchlistService(Context context, ServiceThread handlerThread,

            WatchlistLoggingHandler handler, IIpConnectivityMetrics ipConnectivityMetrics) {

        mContext = context;

        mSettings = WatchlistSettings.getInstance();

        mConfig = WatchlistConfig.getInstance();

        mHandlerThread = handlerThread;

        mNetworkWatchlistHandler = handler;

        mIpConnectivityMetrics = ipConnectivityMetrics;
@@ -228,7 +222,7 @@ public class NetworkWatchlistService extends INetworkWatchlistManager.Stub { 
    public void reloadWatchlist() throws RemoteException {

        enforceWatchlistLoggingPermission();

        Slog.i(TAG, "Reloading watchlist");

        mSettings.reloadSettings();

        mConfig.reloadConfig();

    }



    @Override
@@ -240,7 +234,7 @@ public class NetworkWatchlistService extends INetworkWatchlistManager.Stub { 
    @Override

    protected void dump(FileDescriptor fd, PrintWriter pw, String[] args) {

        if (!DumpUtils.checkDumpPermission(mContext, TAG, pw)) return;

        mSettings.dump(fd, pw, args);

        mConfig.dump(fd, pw, args);

    }



}

services/core/java/com/android/server/net/watchlist/PrivacyUtils.java

0 → 100644
+103 −0
Original line number Diff line number Diff line 
/*

 * Copyright 2017 The Android Open Source Project

 *

 * Licensed under the Apache License, Version 2.0 (the "License");

 * you may not use this file except in compliance with the License.

 * You may obtain a copy of the License at

 *

 *      http://www.apache.org/licenses/LICENSE-2.0

 *

 * Unless required by applicable law or agreed to in writing, software

 * distributed under the License is distributed on an "AS IS" BASIS,

 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

 * See the License for the specific language governing permissions and

 * limitations under the License.

 */



package com.android.server.net.watchlist;



import android.privacy.DifferentialPrivacyEncoder;

import android.privacy.internal.longitudinalreporting.LongitudinalReportingConfig;

import android.privacy.internal.longitudinalreporting.LongitudinalReportingEncoder;



import com.android.internal.annotations.VisibleForTesting;



import java.util.HashMap;

import java.util.List;

import java.util.Map;



/**

 * Helper class to apply differential privacy to watchlist reports.

 */

class PrivacyUtils {



    private static final String TAG = "PrivacyUtils";



    /**

     * Parameters used for encoding watchlist reports.

     * These numbers are optimal parameters for protecting privacy with good utility.

     *

     * TODO: Add links to explain the math behind.

     */

    private static final String ENCODER_ID_PREFIX = "watchlist_encoder:";

    private static final double PROB_F = 0.469;

    private static final double PROB_P = 0.28;

    private static final double PROB_Q = 1.0;



    private PrivacyUtils() {

    }



    /**

     * Get insecure DP encoder.

     * Should not apply it directly on real data as seed is not randomized.

     */

    @VisibleForTesting

    static DifferentialPrivacyEncoder createInsecureDPEncoderForTest(String appDigest) {

        final LongitudinalReportingConfig config = createLongitudinalReportingConfig(appDigest);

        return LongitudinalReportingEncoder.createInsecureEncoderForTest(config);

    }



    /**

     * Get secure encoder to encode watchlist.

     *

     * Warning: If you use the same user secret and app digest, then you will get the same

     * PRR result.

     */

    @VisibleForTesting

    static DifferentialPrivacyEncoder createSecureDPEncoder(byte[] userSecret,

            String appDigest) {

        final LongitudinalReportingConfig config = createLongitudinalReportingConfig(appDigest);

        return LongitudinalReportingEncoder.createEncoder(config, userSecret);

    }



    /**

     * Get DP config for encoding watchlist reports.

     */

    private static LongitudinalReportingConfig createLongitudinalReportingConfig(String appDigest) {

        return new LongitudinalReportingConfig(ENCODER_ID_PREFIX + appDigest, PROB_F, PROB_P,

                PROB_Q);

    }



    /**

     * Create a map that stores appDigest, encoded_visitedWatchlist pairs.

     */

    @VisibleForTesting

    static Map<String, Boolean> createDpEncodedReportMap(boolean isSecure, byte[] userSecret,

            List<String> appDigestList, WatchlistReportDbHelper.AggregatedResult aggregatedResult) {

        final int appDigestListSize = appDigestList.size();

        final HashMap<String, Boolean> resultMap = new HashMap<>(appDigestListSize);

        for (int i = 0; i < appDigestListSize; i++) {

            final String appDigest = appDigestList.get(i);

            // Each app needs to have different PRR result, hence we use appDigest as encoder Id.

            final DifferentialPrivacyEncoder encoder = isSecure

                    ? createSecureDPEncoder(userSecret, appDigest)

                    : createInsecureDPEncoderForTest(appDigest);

            final boolean visitedWatchlist = aggregatedResult.appDigestList.contains(appDigest);

            // Get the least significant bit of first byte, and set result to True if it is 1

            boolean encodedVisitedWatchlist = ((int) encoder.encodeBoolean(visitedWatchlist)[0]

                    & 0x1) == 0x1;

            resultMap.put(appDigest, encodedVisitedWatchlist);

        }

        return resultMap;

    }

}

services/core/java/com/android/server/net/watchlist/ReportEncoder.java

0 → 100644
+126 −0
Original line number Diff line number Diff line 
/*

 * Copyright 2017 The Android Open Source Project

 *

 * Licensed under the Apache License, Version 2.0 (the "License");

 * you may not use this file except in compliance with the License.

 * You may obtain a copy of the License at

 *

 *      http://www.apache.org/licenses/LICENSE-2.0

 *

 * Unless required by applicable law or agreed to in writing, software

 * distributed under the License is distributed on an "AS IS" BASIS,

 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

 * See the License for the specific language governing permissions and

 * limitations under the License.

 */





package com.android.server.net.watchlist;



import android.annotation.Nullable;

import android.util.Log;



import com.android.internal.annotations.VisibleForTesting;

import com.android.internal.util.HexDump;



import java.util.ArrayList;

import java.util.Collections;

import java.util.List;

import java.util.Map;



/**

 * Helper class to encode and generate serialized DP encoded watchlist report.

 *

 * <p>Serialized report data structure:

 * [4 bytes magic number][4_bytes_report_version_code][32_bytes_watchlist_hash]

 * [app_1_digest_byte_array][app_1_encoded_visited_cnc_byte]

 * [app_2_digest_byte_array][app_2_encoded_visited_cnc_byte]

 * ...

 *

 * Total size: 4 + 4 + 32 + (32+1)*N, where N = number of digests

 */

class ReportEncoder {



    private static final String TAG = "ReportEncoder";



    // Report header magic number

    private static final byte[] MAGIC_NUMBER = {(byte) 0x8D, (byte) 0x37, (byte) 0x0A, (byte) 0xAC};

    // Report version number, as file format / parameters can be changed in later version, we need

    // to have versioning on watchlist report format

    private static final byte[] REPORT_VERSION = {(byte) 0x00, (byte) 0x01};



    private static final int WATCHLIST_HASH_SIZE = 32;

    private static final int APP_DIGEST_SIZE = 32;



    /**

     * Apply DP on watchlist results, and generate a serialized watchlist report ready to store

     * in DropBox.

     */

    static byte[] encodeWatchlistReport(WatchlistConfig config, byte[] userSecret,

            List<String> appDigestList, WatchlistReportDbHelper.AggregatedResult aggregatedResult) {

        Map<String, Boolean> resultMap = PrivacyUtils.createDpEncodedReportMap(

                config.isConfigSecure(), userSecret, appDigestList, aggregatedResult);

        return serializeReport(config, resultMap);

    }



    /**

     * Convert DP encoded watchlist report into byte[] format.

     * TODO: Serialize it using protobuf

     *

     * @param encodedReportMap DP encoded watchlist report.

     * @return Watchlist report in byte[] format, which will be shared in Dropbox. Null if

     * watchlist report cannot be generated.

     */

    @Nullable

    @VisibleForTesting

    static byte[] serializeReport(WatchlistConfig config,

            Map<String, Boolean> encodedReportMap) {

        // TODO: Handle watchlist config changed case

        final byte[] watchlistHash = config.getWatchlistConfigHash();

        if (watchlistHash == null) {

            Log.e(TAG, "No watchlist hash");

            return null;

        }

        if (watchlistHash.length != WATCHLIST_HASH_SIZE) {

            Log.e(TAG, "Unexpected hash length");

            return null;

        }

        final int reportMapSize = encodedReportMap.size();

        final byte[] outputReport =

                new byte[MAGIC_NUMBER.length + REPORT_VERSION.length + WATCHLIST_HASH_SIZE

                        + reportMapSize * (APP_DIGEST_SIZE + /* Result */ 1)];

        final List<String> sortedKeys = new ArrayList(encodedReportMap.keySet());

        Collections.sort(sortedKeys);



        int offset = 0;



        // Set magic number to report

        System.arraycopy(MAGIC_NUMBER, 0, outputReport, offset, MAGIC_NUMBER.length);

        offset += MAGIC_NUMBER.length;



        // Set report version to report

        System.arraycopy(REPORT_VERSION, 0, outputReport, offset, REPORT_VERSION.length);

        offset += REPORT_VERSION.length;



        // Set watchlist hash to report

        System.arraycopy(watchlistHash, 0, outputReport, offset, watchlistHash.length);

        offset += watchlistHash.length;



        // Set app digest, encoded_isPha pair to report

        for (int i = 0; i < reportMapSize; i++) {

            String key = sortedKeys.get(i);

            byte[] digest = HexDump.hexStringToByteArray(key);

            boolean isPha = encodedReportMap.get(key);

            System.arraycopy(digest, 0, outputReport, offset, APP_DIGEST_SIZE);

            offset += digest.length;

            outputReport[offset] = (byte) (isPha ? 1 : 0);

            offset += 1;

        }

        if (outputReport.length != offset) {

            Log.e(TAG, "Watchlist report size does not match! Offset: " + offset + ", report size: "

                    + outputReport.length);



        }

        return outputReport;

    }

}

services/core/java/com/android/server/net/watchlist/WatchlistConfig.java

0 → 100644
+245 −0
Original line number Diff line number Diff line 
/*

 * Copyright (C) 2017 The Android Open Source Project

 *

 * Licensed under the Apache License, Version 2.0 (the "License");

 * you may not use this file except in compliance with the License.

 * You may obtain a copy of the License at

 *

 *      http://www.apache.org/licenses/LICENSE-2.0

 *

 * Unless required by applicable law or agreed to in writing, software

 * distributed under the License is distributed on an "AS IS" BASIS,

 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

 * See the License for the specific language governing permissions and

 * limitations under the License.

 */



package com.android.server.net.watchlist;



import android.util.AtomicFile;

import android.util.Log;

import android.util.Slog;

import android.util.Xml;



import com.android.internal.annotations.VisibleForTesting;

import com.android.internal.util.HexDump;

import com.android.internal.util.XmlUtils;



import org.xmlpull.v1.XmlPullParser;

import org.xmlpull.v1.XmlPullParserException;



import java.io.File;

import java.io.FileDescriptor;

import java.io.FileInputStream;

import java.io.IOException;

import java.io.PrintWriter;

import java.nio.charset.StandardCharsets;

import java.security.MessageDigest;

import java.security.NoSuchAlgorithmException;

import java.util.ArrayList;

import java.util.List;

import java.util.zip.CRC32;



/**

 * Class for watchlist config operations, like setting watchlist, query if a domain

 * exists in watchlist.

 */

class WatchlistConfig {

    private static final String TAG = "WatchlistConfig";



    // Watchlist config that pushed by ConfigUpdater.

    private static final String NETWORK_WATCHLIST_DB_PATH =

            "/data/misc/network_watchlist/network_watchlist.xml";



    // Hash for null / unknown config, a 32 byte array filled with content 0x00

    private static final byte[] UNKNOWN_CONFIG_HASH = new byte[32];



    private static class XmlTags {

        private static final String WATCHLIST_CONFIG = "watchlist-config";

        private static final String SHA256_DOMAIN = "sha256-domain";

        private static final String CRC32_DOMAIN = "crc32-domain";

        private static final String SHA256_IP = "sha256-ip";

        private static final String CRC32_IP = "crc32-ip";

        private static final String HASH = "hash";

    }



    private static class CrcShaDigests {

        final HarmfulDigests crc32Digests;

        final HarmfulDigests sha256Digests;



        public CrcShaDigests(HarmfulDigests crc32Digests, HarmfulDigests sha256Digests) {

            this.crc32Digests = crc32Digests;

            this.sha256Digests = sha256Digests;

        }

    }



    /*

     * This is always true unless watchlist is being set by adb command, then it will be false

     * until next reboot.

     */

    private boolean mIsSecureConfig = true;



    private final static WatchlistConfig sInstance = new WatchlistConfig();

    private final File mXmlFile;



    private volatile CrcShaDigests mDomainDigests;

    private volatile CrcShaDigests mIpDigests;



    public static WatchlistConfig getInstance() {

        return sInstance;

    }



    private WatchlistConfig() {

        this(new File(NETWORK_WATCHLIST_DB_PATH));

    }



    @VisibleForTesting

    protected WatchlistConfig(File xmlFile) {

        mXmlFile = xmlFile;

        reloadConfig();

    }



    /**

     * Reload watchlist by reading config file.

     */

    public void reloadConfig() {

        try (FileInputStream stream = new FileInputStream(mXmlFile)){

            final List<byte[]> crc32DomainList = new ArrayList<>();

            final List<byte[]> sha256DomainList = new ArrayList<>();

            final List<byte[]> crc32IpList = new ArrayList<>();

            final List<byte[]> sha256IpList = new ArrayList<>();



            XmlPullParser parser = Xml.newPullParser();

            parser.setInput(stream, StandardCharsets.UTF_8.name());

            parser.nextTag();

            parser.require(XmlPullParser.START_TAG, null, XmlTags.WATCHLIST_CONFIG);

            while (parser.nextTag() == XmlPullParser.START_TAG) {

                String tagName = parser.getName();

                switch (tagName) {

                    case XmlTags.CRC32_DOMAIN:

                        parseHashes(parser, tagName, crc32DomainList);

                        break;

                    case XmlTags.CRC32_IP:

                        parseHashes(parser, tagName, crc32IpList);

                        break;

                    case XmlTags.SHA256_DOMAIN:

                        parseHashes(parser, tagName, sha256DomainList);

                        break;

                    case XmlTags.SHA256_IP:

                        parseHashes(parser, tagName, sha256IpList);

                        break;

                    default:

                        Log.w(TAG, "Unknown element: " + parser.getName());

                        XmlUtils.skipCurrentTag(parser);

                }

            }

            parser.require(XmlPullParser.END_TAG, null, XmlTags.WATCHLIST_CONFIG);

            mDomainDigests = new CrcShaDigests(new HarmfulDigests(crc32DomainList),

                    new HarmfulDigests(sha256DomainList));

            mIpDigests = new CrcShaDigests(new HarmfulDigests(crc32IpList),

                    new HarmfulDigests(sha256IpList));

            Log.i(TAG, "Reload watchlist done");

        } catch (IllegalStateException | NullPointerException | NumberFormatException |

                XmlPullParserException | IOException | IndexOutOfBoundsException e) {

            Slog.e(TAG, "Failed parsing xml", e);

        }

    }



    private void parseHashes(XmlPullParser parser, String tagName, List<byte[]> hashList)

            throws IOException, XmlPullParserException {

        parser.require(XmlPullParser.START_TAG, null, tagName);

        // Get all the hashes for this tag

        while (parser.nextTag() == XmlPullParser.START_TAG) {

            parser.require(XmlPullParser.START_TAG, null, XmlTags.HASH);

            byte[] hash = HexDump.hexStringToByteArray(parser.nextText());

            parser.require(XmlPullParser.END_TAG, null, XmlTags.HASH);

            hashList.add(hash);

        }

        parser.require(XmlPullParser.END_TAG, null, tagName);

    }



    public boolean containsDomain(String domain) {

        final CrcShaDigests domainDigests = mDomainDigests;

        if (domainDigests == null) {

            Slog.wtf(TAG, "domainDigests should not be null");

            return false;

        }

        // First it does a quick CRC32 check.

        final byte[] crc32 = getCrc32(domain);

        if (!domainDigests.crc32Digests.contains(crc32)) {

            return false;

        }

        // Now we do a slow SHA256 check.

        final byte[] sha256 = getSha256(domain);

        return domainDigests.sha256Digests.contains(sha256);

    }



    public boolean containsIp(String ip) {

        final CrcShaDigests ipDigests = mIpDigests;

        if (ipDigests == null) {

            Slog.wtf(TAG, "ipDigests should not be null");

            return false;

        }

        // First it does a quick CRC32 check.

        final byte[] crc32 = getCrc32(ip);

        if (!ipDigests.crc32Digests.contains(crc32)) {

            return false;

        }

        // Now we do a slow SHA256 check.

        final byte[] sha256 = getSha256(ip);

        return ipDigests.sha256Digests.contains(sha256);

    }





    /** Get CRC32 of a string

     *

     * TODO: Review if we should use CRC32 or other algorithms

     */

    private byte[] getCrc32(String str) {

        final CRC32 crc = new CRC32();

        crc.update(str.getBytes());

        final long tmp = crc.getValue();

        return new byte[]{(byte) (tmp >> 24 & 255), (byte) (tmp >> 16 & 255),

                (byte) (tmp >> 8 & 255), (byte) (tmp & 255)};

    }



    /** Get SHA256 of a string */

    private byte[] getSha256(String str) {

        MessageDigest messageDigest;

        try {

            messageDigest = MessageDigest.getInstance("SHA256");

        } catch (NoSuchAlgorithmException e) {

            /* can't happen */

            return null;

        }

        messageDigest.update(str.getBytes());

        return messageDigest.digest();

    }



    public boolean isConfigSecure() {

        return mIsSecureConfig;

    }



    public byte[] getWatchlistConfigHash() {

        if (!mXmlFile.exists()) {

            return UNKNOWN_CONFIG_HASH;

        }

        try {

            return DigestUtils.getSha256Hash(mXmlFile);

        } catch (IOException | NoSuchAlgorithmException e) {

            Log.e(TAG, "Unable to get watchlist config hash", e);

        }

        return UNKNOWN_CONFIG_HASH;

    }



    public void dump(FileDescriptor fd, PrintWriter pw, String[] args) {

        pw.println("Domain CRC32 digest list:");

        mDomainDigests.crc32Digests.dump(fd, pw, args);

        pw.println("Domain SHA256 digest list:");

        mDomainDigests.sha256Digests.dump(fd, pw, args);

        pw.println("Ip CRC32 digest list:");

        mIpDigests.crc32Digests.dump(fd, pw, args);

        pw.println("Ip SHA256 digest list:");

        mIpDigests.sha256Digests.dump(fd, pw, args);

    }

}

services/core/java/com/android/server/net/watchlist/WatchlistLoggingHandler.java

+81 −50

File changed.

Preview size limit exceeded, changes collapsed.