Merge "Apply differential privacy on watchlist report" (d61f7faf) · Commits · e / os / android_frameworks_base

services/core/java/com/android/server/net/watchlist/NetworkWatchlistService.java

+5 −11

Original line number	Diff line number	Diff line
		@@ -17,15 +17,11 @@
		package com.android.server.net.watchlist;

		import android.content.Context;
		import android.content.pm.PackageManager;
		import android.content.pm.PackageManager.NameNotFoundException;
		import android.net.IIpConnectivityMetrics;
		import android.net.INetdEventCallback;
		import android.net.NetworkWatchlistManager;
		import android.net.metrics.IpConnectivityLog;
		import android.os.Binder;
		import android.os.Process;
		import android.os.SharedMemory;
		import android.os.RemoteException;
		import android.os.ServiceManager;
		import android.os.SystemProperties;
		@@ -42,9 +38,7 @@ import com.android.server.ServiceThread;
		import com.android.server.SystemService;

		import java.io.FileDescriptor;
		import java.io.IOException;
		import java.io.PrintWriter;
		import java.util.List;

		/**
		* Implementation of network watchlist service.
		@@ -99,7 +93,7 @@ public class NetworkWatchlistService extends INetworkWatchlistManager.Stub {
		private volatile boolean mIsLoggingEnabled = false;
		private final Object mLoggingSwitchLock = new Object();

		private final WatchlistSettings mSettings;
		private final WatchlistConfig mConfig;
		private final Context mContext;

		// Separate thread to handle expensive watchlist logging work.
		@@ -112,7 +106,7 @@ public class NetworkWatchlistService extends INetworkWatchlistManager.Stub {

		public NetworkWatchlistService(Context context) {
		mContext = context;
		mSettings = WatchlistSettings.getInstance();
		mConfig = WatchlistConfig.getInstance();
		mHandlerThread = new ServiceThread(TAG, Process.THREAD_PRIORITY_BACKGROUND,
		/* allowIo */ false);
		mHandlerThread.start();
		@@ -126,7 +120,7 @@ public class NetworkWatchlistService extends INetworkWatchlistManager.Stub {
		NetworkWatchlistService(Context context, ServiceThread handlerThread,
		WatchlistLoggingHandler handler, IIpConnectivityMetrics ipConnectivityMetrics) {
		mContext = context;
		mSettings = WatchlistSettings.getInstance();
		mConfig = WatchlistConfig.getInstance();
		mHandlerThread = handlerThread;
		mNetworkWatchlistHandler = handler;
		mIpConnectivityMetrics = ipConnectivityMetrics;
		@@ -228,7 +222,7 @@ public class NetworkWatchlistService extends INetworkWatchlistManager.Stub {
		public void reloadWatchlist() throws RemoteException {
		enforceWatchlistLoggingPermission();
		Slog.i(TAG, "Reloading watchlist");
		mSettings.reloadSettings();
		mConfig.reloadConfig();
		}

		@Override
		@@ -240,7 +234,7 @@ public class NetworkWatchlistService extends INetworkWatchlistManager.Stub {
		@Override
		protected void dump(FileDescriptor fd, PrintWriter pw, String[] args) {
		if (!DumpUtils.checkDumpPermission(mContext, TAG, pw)) return;
		mSettings.dump(fd, pw, args);
		mConfig.dump(fd, pw, args);
		}

		}

services/core/java/com/android/server/net/watchlist/PrivacyUtils.java

0 → 100644

+103 −0

Original line number	Diff line number	Diff line
		/*
		* Copyright 2017 The Android Open Source Project
		*
		* Licensed under the Apache License, Version 2.0 (the "License");
		* you may not use this file except in compliance with the License.
		* You may obtain a copy of the License at
		*
		* http://www.apache.org/licenses/LICENSE-2.0
		*
		* Unless required by applicable law or agreed to in writing, software
		* distributed under the License is distributed on an "AS IS" BASIS,
		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
		* See the License for the specific language governing permissions and
		* limitations under the License.
		*/

		package com.android.server.net.watchlist;

		import android.privacy.DifferentialPrivacyEncoder;
		import android.privacy.internal.longitudinalreporting.LongitudinalReportingConfig;
		import android.privacy.internal.longitudinalreporting.LongitudinalReportingEncoder;

		import com.android.internal.annotations.VisibleForTesting;

		import java.util.HashMap;
		import java.util.List;
		import java.util.Map;

		/**
		* Helper class to apply differential privacy to watchlist reports.
		*/
		class PrivacyUtils {

		private static final String TAG = "PrivacyUtils";

		/**
		* Parameters used for encoding watchlist reports.
		* These numbers are optimal parameters for protecting privacy with good utility.
		*
		* TODO: Add links to explain the math behind.
		*/
		private static final String ENCODER_ID_PREFIX = "watchlist_encoder:";
		private static final double PROB_F = 0.469;
		private static final double PROB_P = 0.28;
		private static final double PROB_Q = 1.0;

		private PrivacyUtils() {
		}

		/**
		* Get insecure DP encoder.
		* Should not apply it directly on real data as seed is not randomized.
		*/
		@VisibleForTesting
		static DifferentialPrivacyEncoder createInsecureDPEncoderForTest(String appDigest) {
		final LongitudinalReportingConfig config = createLongitudinalReportingConfig(appDigest);
		return LongitudinalReportingEncoder.createInsecureEncoderForTest(config);
		}

		/**
		* Get secure encoder to encode watchlist.
		*
		* Warning: If you use the same user secret and app digest, then you will get the same
		* PRR result.
		*/
		@VisibleForTesting
		static DifferentialPrivacyEncoder createSecureDPEncoder(byte[] userSecret,
		String appDigest) {
		final LongitudinalReportingConfig config = createLongitudinalReportingConfig(appDigest);
		return LongitudinalReportingEncoder.createEncoder(config, userSecret);
		}

		/**
		* Get DP config for encoding watchlist reports.
		*/
		private static LongitudinalReportingConfig createLongitudinalReportingConfig(String appDigest) {
		return new LongitudinalReportingConfig(ENCODER_ID_PREFIX + appDigest, PROB_F, PROB_P,
		PROB_Q);
		}

		/**
		* Create a map that stores appDigest, encoded_visitedWatchlist pairs.
		*/
		@VisibleForTesting
		static Map<String, Boolean> createDpEncodedReportMap(boolean isSecure, byte[] userSecret,
		List<String> appDigestList, WatchlistReportDbHelper.AggregatedResult aggregatedResult) {
		final int appDigestListSize = appDigestList.size();
		final HashMap<String, Boolean> resultMap = new HashMap<>(appDigestListSize);
		for (int i = 0; i < appDigestListSize; i++) {
		final String appDigest = appDigestList.get(i);
		// Each app needs to have different PRR result, hence we use appDigest as encoder Id.
		final DifferentialPrivacyEncoder encoder = isSecure
		? createSecureDPEncoder(userSecret, appDigest)
		: createInsecureDPEncoderForTest(appDigest);
		final boolean visitedWatchlist = aggregatedResult.appDigestList.contains(appDigest);
		// Get the least significant bit of first byte, and set result to True if it is 1
		boolean encodedVisitedWatchlist = ((int) encoder.encodeBoolean(visitedWatchlist)[0]
		& 0x1) == 0x1;
		resultMap.put(appDigest, encodedVisitedWatchlist);
		}
		return resultMap;
		}
		}

services/core/java/com/android/server/net/watchlist/ReportEncoder.java

0 → 100644

+126 −0

Original line number	Diff line number	Diff line
		/*
		* Copyright 2017 The Android Open Source Project
		*
		* Licensed under the Apache License, Version 2.0 (the "License");
		* you may not use this file except in compliance with the License.
		* You may obtain a copy of the License at
		*
		* http://www.apache.org/licenses/LICENSE-2.0
		*
		* Unless required by applicable law or agreed to in writing, software
		* distributed under the License is distributed on an "AS IS" BASIS,
		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
		* See the License for the specific language governing permissions and
		* limitations under the License.
		*/


		package com.android.server.net.watchlist;

		import android.annotation.Nullable;
		import android.util.Log;

		import com.android.internal.annotations.VisibleForTesting;
		import com.android.internal.util.HexDump;

		import java.util.ArrayList;
		import java.util.Collections;
		import java.util.List;
		import java.util.Map;

		/**
		* Helper class to encode and generate serialized DP encoded watchlist report.
		*
		* <p>Serialized report data structure:
		* [4 bytes magic number][4_bytes_report_version_code][32_bytes_watchlist_hash]
		* [app_1_digest_byte_array][app_1_encoded_visited_cnc_byte]
		* [app_2_digest_byte_array][app_2_encoded_visited_cnc_byte]
		* ...
		*
		* Total size: 4 + 4 + 32 + (32+1)*N, where N = number of digests
		*/
		class ReportEncoder {

		private static final String TAG = "ReportEncoder";

		// Report header magic number
		private static final byte[] MAGIC_NUMBER = {(byte) 0x8D, (byte) 0x37, (byte) 0x0A, (byte) 0xAC};
		// Report version number, as file format / parameters can be changed in later version, we need
		// to have versioning on watchlist report format
		private static final byte[] REPORT_VERSION = {(byte) 0x00, (byte) 0x01};

		private static final int WATCHLIST_HASH_SIZE = 32;
		private static final int APP_DIGEST_SIZE = 32;

		/**
		* Apply DP on watchlist results, and generate a serialized watchlist report ready to store
		* in DropBox.
		*/
		static byte[] encodeWatchlistReport(WatchlistConfig config, byte[] userSecret,
		List<String> appDigestList, WatchlistReportDbHelper.AggregatedResult aggregatedResult) {
		Map<String, Boolean> resultMap = PrivacyUtils.createDpEncodedReportMap(
		config.isConfigSecure(), userSecret, appDigestList, aggregatedResult);
		return serializeReport(config, resultMap);
		}

		/**
		* Convert DP encoded watchlist report into byte[] format.
		* TODO: Serialize it using protobuf
		*
		* @param encodedReportMap DP encoded watchlist report.
		* @return Watchlist report in byte[] format, which will be shared in Dropbox. Null if
		* watchlist report cannot be generated.
		*/
		@Nullable
		@VisibleForTesting
		static byte[] serializeReport(WatchlistConfig config,
		Map<String, Boolean> encodedReportMap) {
		// TODO: Handle watchlist config changed case
		final byte[] watchlistHash = config.getWatchlistConfigHash();
		if (watchlistHash == null) {
		Log.e(TAG, "No watchlist hash");
		return null;
		}
		if (watchlistHash.length != WATCHLIST_HASH_SIZE) {
		Log.e(TAG, "Unexpected hash length");
		return null;
		}
		final int reportMapSize = encodedReportMap.size();
		final byte[] outputReport =
		new byte[MAGIC_NUMBER.length + REPORT_VERSION.length + WATCHLIST_HASH_SIZE
		+ reportMapSize * (APP_DIGEST_SIZE + /* Result */ 1)];
		final List<String> sortedKeys = new ArrayList(encodedReportMap.keySet());
		Collections.sort(sortedKeys);

		int offset = 0;

		// Set magic number to report
		System.arraycopy(MAGIC_NUMBER, 0, outputReport, offset, MAGIC_NUMBER.length);
		offset += MAGIC_NUMBER.length;

		// Set report version to report
		System.arraycopy(REPORT_VERSION, 0, outputReport, offset, REPORT_VERSION.length);
		offset += REPORT_VERSION.length;

		// Set watchlist hash to report
		System.arraycopy(watchlistHash, 0, outputReport, offset, watchlistHash.length);
		offset += watchlistHash.length;

		// Set app digest, encoded_isPha pair to report
		for (int i = 0; i < reportMapSize; i++) {
		String key = sortedKeys.get(i);
		byte[] digest = HexDump.hexStringToByteArray(key);
		boolean isPha = encodedReportMap.get(key);
		System.arraycopy(digest, 0, outputReport, offset, APP_DIGEST_SIZE);
		offset += digest.length;
		outputReport[offset] = (byte) (isPha ? 1 : 0);
		offset += 1;
		}
		if (outputReport.length != offset) {
		Log.e(TAG, "Watchlist report size does not match! Offset: " + offset + ", report size: "
		+ outputReport.length);

		}
		return outputReport;
		}
		}

services/core/java/com/android/server/net/watchlist/WatchlistConfig.java

0 → 100644

+245 −0

Original line number	Diff line number	Diff line
		/*
		* Copyright (C) 2017 The Android Open Source Project
		*
		* Licensed under the Apache License, Version 2.0 (the "License");
		* you may not use this file except in compliance with the License.
		* You may obtain a copy of the License at
		*
		* http://www.apache.org/licenses/LICENSE-2.0
		*
		* Unless required by applicable law or agreed to in writing, software
		* distributed under the License is distributed on an "AS IS" BASIS,
		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
		* See the License for the specific language governing permissions and
		* limitations under the License.
		*/

		package com.android.server.net.watchlist;

		import android.util.AtomicFile;
		import android.util.Log;
		import android.util.Slog;
		import android.util.Xml;

		import com.android.internal.annotations.VisibleForTesting;
		import com.android.internal.util.HexDump;
		import com.android.internal.util.XmlUtils;

		import org.xmlpull.v1.XmlPullParser;
		import org.xmlpull.v1.XmlPullParserException;

		import java.io.File;
		import java.io.FileDescriptor;
		import java.io.FileInputStream;
		import java.io.IOException;
		import java.io.PrintWriter;
		import java.nio.charset.StandardCharsets;
		import java.security.MessageDigest;
		import java.security.NoSuchAlgorithmException;
		import java.util.ArrayList;
		import java.util.List;
		import java.util.zip.CRC32;

		/**
		* Class for watchlist config operations, like setting watchlist, query if a domain
		* exists in watchlist.
		*/
		class WatchlistConfig {
		private static final String TAG = "WatchlistConfig";

		// Watchlist config that pushed by ConfigUpdater.
		private static final String NETWORK_WATCHLIST_DB_PATH =
		"/data/misc/network_watchlist/network_watchlist.xml";

		// Hash for null / unknown config, a 32 byte array filled with content 0x00
		private static final byte[] UNKNOWN_CONFIG_HASH = new byte[32];

		private static class XmlTags {
		private static final String WATCHLIST_CONFIG = "watchlist-config";
		private static final String SHA256_DOMAIN = "sha256-domain";
		private static final String CRC32_DOMAIN = "crc32-domain";
		private static final String SHA256_IP = "sha256-ip";
		private static final String CRC32_IP = "crc32-ip";
		private static final String HASH = "hash";
		}

		private static class CrcShaDigests {
		final HarmfulDigests crc32Digests;
		final HarmfulDigests sha256Digests;

		public CrcShaDigests(HarmfulDigests crc32Digests, HarmfulDigests sha256Digests) {
		this.crc32Digests = crc32Digests;
		this.sha256Digests = sha256Digests;
		}
		}

		/*
		* This is always true unless watchlist is being set by adb command, then it will be false
		* until next reboot.
		*/
		private boolean mIsSecureConfig = true;

		private final static WatchlistConfig sInstance = new WatchlistConfig();
		private final File mXmlFile;

		private volatile CrcShaDigests mDomainDigests;
		private volatile CrcShaDigests mIpDigests;

		public static WatchlistConfig getInstance() {
		return sInstance;
		}

		private WatchlistConfig() {
		this(new File(NETWORK_WATCHLIST_DB_PATH));
		}

		@VisibleForTesting
		protected WatchlistConfig(File xmlFile) {
		mXmlFile = xmlFile;
		reloadConfig();
		}

		/**
		* Reload watchlist by reading config file.
		*/
		public void reloadConfig() {
		try (FileInputStream stream = new FileInputStream(mXmlFile)){
		final List<byte[]> crc32DomainList = new ArrayList<>();
		final List<byte[]> sha256DomainList = new ArrayList<>();
		final List<byte[]> crc32IpList = new ArrayList<>();
		final List<byte[]> sha256IpList = new ArrayList<>();

		XmlPullParser parser = Xml.newPullParser();
		parser.setInput(stream, StandardCharsets.UTF_8.name());
		parser.nextTag();
		parser.require(XmlPullParser.START_TAG, null, XmlTags.WATCHLIST_CONFIG);
		while (parser.nextTag() == XmlPullParser.START_TAG) {
		String tagName = parser.getName();
		switch (tagName) {
		case XmlTags.CRC32_DOMAIN:
		parseHashes(parser, tagName, crc32DomainList);
		break;
		case XmlTags.CRC32_IP:
		parseHashes(parser, tagName, crc32IpList);
		break;
		case XmlTags.SHA256_DOMAIN:
		parseHashes(parser, tagName, sha256DomainList);
		break;
		case XmlTags.SHA256_IP:
		parseHashes(parser, tagName, sha256IpList);
		break;
		default:
		Log.w(TAG, "Unknown element: " + parser.getName());
		XmlUtils.skipCurrentTag(parser);
		}
		}
		parser.require(XmlPullParser.END_TAG, null, XmlTags.WATCHLIST_CONFIG);
		mDomainDigests = new CrcShaDigests(new HarmfulDigests(crc32DomainList),
		new HarmfulDigests(sha256DomainList));
		mIpDigests = new CrcShaDigests(new HarmfulDigests(crc32IpList),
		new HarmfulDigests(sha256IpList));
		Log.i(TAG, "Reload watchlist done");
		} catch (IllegalStateException \| NullPointerException \| NumberFormatException \|
		XmlPullParserException \| IOException \| IndexOutOfBoundsException e) {
		Slog.e(TAG, "Failed parsing xml", e);
		}
		}

		private void parseHashes(XmlPullParser parser, String tagName, List<byte[]> hashList)
		throws IOException, XmlPullParserException {
		parser.require(XmlPullParser.START_TAG, null, tagName);
		// Get all the hashes for this tag
		while (parser.nextTag() == XmlPullParser.START_TAG) {
		parser.require(XmlPullParser.START_TAG, null, XmlTags.HASH);
		byte[] hash = HexDump.hexStringToByteArray(parser.nextText());
		parser.require(XmlPullParser.END_TAG, null, XmlTags.HASH);
		hashList.add(hash);
		}
		parser.require(XmlPullParser.END_TAG, null, tagName);
		}

		public boolean containsDomain(String domain) {
		final CrcShaDigests domainDigests = mDomainDigests;
		if (domainDigests == null) {
		Slog.wtf(TAG, "domainDigests should not be null");
		return false;
		}
		// First it does a quick CRC32 check.
		final byte[] crc32 = getCrc32(domain);
		if (!domainDigests.crc32Digests.contains(crc32)) {
		return false;
		}
		// Now we do a slow SHA256 check.
		final byte[] sha256 = getSha256(domain);
		return domainDigests.sha256Digests.contains(sha256);
		}

		public boolean containsIp(String ip) {
		final CrcShaDigests ipDigests = mIpDigests;
		if (ipDigests == null) {
		Slog.wtf(TAG, "ipDigests should not be null");
		return false;
		}
		// First it does a quick CRC32 check.
		final byte[] crc32 = getCrc32(ip);
		if (!ipDigests.crc32Digests.contains(crc32)) {
		return false;
		}
		// Now we do a slow SHA256 check.
		final byte[] sha256 = getSha256(ip);
		return ipDigests.sha256Digests.contains(sha256);
		}


		/** Get CRC32 of a string
		*
		* TODO: Review if we should use CRC32 or other algorithms
		*/
		private byte[] getCrc32(String str) {
		final CRC32 crc = new CRC32();
		crc.update(str.getBytes());
		final long tmp = crc.getValue();
		return new byte[]{(byte) (tmp >> 24 & 255), (byte) (tmp >> 16 & 255),
		(byte) (tmp >> 8 & 255), (byte) (tmp & 255)};
		}

		/** Get SHA256 of a string */
		private byte[] getSha256(String str) {
		MessageDigest messageDigest;
		try {
		messageDigest = MessageDigest.getInstance("SHA256");
		} catch (NoSuchAlgorithmException e) {
		/* can't happen */
		return null;
		}
		messageDigest.update(str.getBytes());
		return messageDigest.digest();
		}

		public boolean isConfigSecure() {
		return mIsSecureConfig;
		}

		public byte[] getWatchlistConfigHash() {
		if (!mXmlFile.exists()) {
		return UNKNOWN_CONFIG_HASH;
		}
		try {
		return DigestUtils.getSha256Hash(mXmlFile);
		} catch (IOException \| NoSuchAlgorithmException e) {
		Log.e(TAG, "Unable to get watchlist config hash", e);
		}
		return UNKNOWN_CONFIG_HASH;
		}

		public void dump(FileDescriptor fd, PrintWriter pw, String[] args) {
		pw.println("Domain CRC32 digest list:");
		mDomainDigests.crc32Digests.dump(fd, pw, args);
		pw.println("Domain SHA256 digest list:");
		mDomainDigests.sha256Digests.dump(fd, pw, args);
		pw.println("Ip CRC32 digest list:");
		mIpDigests.crc32Digests.dump(fd, pw, args);
		pw.println("Ip SHA256 digest list:");
		mIpDigests.sha256Digests.dump(fd, pw, args);
		}
		}

services/core/java/com/android/server/net/watchlist/WatchlistLoggingHandler.java

+81 −50

File changed.

Preview size limit exceeded, changes collapsed.