Ports chunking/cdc from gmscore to AOSP. (86500337) · Commits · e / os / android_frameworks_base

services/backup/java/com/android/server/backup/encryption/chunking/cdc/ContentDefinedChunker.java

0 → 100644

+136 −0

Original line number	Diff line number	Diff line
		/*
		* Copyright (C) 2018 The Android Open Source Project
		*
		* Licensed under the Apache License, Version 2.0 (the "License");
		* you may not use this file except in compliance with the License.
		* You may obtain a copy of the License at
		*
		* http://www.apache.org/licenses/LICENSE-2.0
		*
		* Unless required by applicable law or agreed to in writing, software
		* distributed under the License is distributed on an "AS IS" BASIS,
		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
		* See the License for the specific language governing permissions and
		* limitations under the License.
		*/

		package com.android.server.backup.encryption.chunking.cdc;

		import static com.android.internal.util.Preconditions.checkArgument;

		import com.android.server.backup.encryption.chunking.Chunker;

		import java.io.IOException;
		import java.io.InputStream;
		import java.security.GeneralSecurityException;
		import java.util.Arrays;

		/** Splits a stream of bytes into variable-sized chunks, using content-defined chunking. */
		public class ContentDefinedChunker implements Chunker {
		private static final int WINDOW_SIZE = 31;
		private static final byte DEFAULT_OUT_BYTE = (byte) 0;

		private final byte[] mChunkBuffer;
		private final RabinFingerprint64 mRabinFingerprint64;
		private final FingerprintMixer mFingerprintMixer;
		private final BreakpointPredicate mBreakpointPredicate;
		private final int mMinChunkSize;
		private final int mMaxChunkSize;

		/**
		* Constructor.
		*
		* @param minChunkSize The minimum size of a chunk. No chunk will be produced of a size smaller
		* than this except possibly at the very end of the stream.
		* @param maxChunkSize The maximum size of a chunk. No chunk will be produced of a larger size.
		* @param rabinFingerprint64 Calculates fingerprints, with which to determine breakpoints.
		* @param breakpointPredicate Given a Rabin fingerprint, returns whether this ought to be a
		* breakpoint.
		*/
		public ContentDefinedChunker(
		int minChunkSize,
		int maxChunkSize,
		RabinFingerprint64 rabinFingerprint64,
		FingerprintMixer fingerprintMixer,
		BreakpointPredicate breakpointPredicate) {
		checkArgument(
		minChunkSize >= WINDOW_SIZE,
		"Minimum chunk size must be greater than window size.");
		checkArgument(
		maxChunkSize >= minChunkSize,
		"Maximum chunk size cannot be smaller than minimum chunk size.");
		mChunkBuffer = new byte[maxChunkSize];
		mRabinFingerprint64 = rabinFingerprint64;
		mBreakpointPredicate = breakpointPredicate;
		mFingerprintMixer = fingerprintMixer;
		mMinChunkSize = minChunkSize;
		mMaxChunkSize = maxChunkSize;
		}

		/**
		* Breaks the input stream into variable-sized chunks.
		*
		* @param inputStream The input bytes to break into chunks.
		* @param chunkConsumer A function to process each chunk as it's generated.
		* @throws IOException Thrown if there is an issue reading from the input stream.
		* @throws GeneralSecurityException Thrown if the {@link ChunkConsumer} throws it.
		*/
		@Override
		public void chunkify(InputStream inputStream, ChunkConsumer chunkConsumer)
		throws IOException, GeneralSecurityException {
		int chunkLength;
		int initialReadLength = mMinChunkSize - WINDOW_SIZE;

		// Performance optimization - there is no reason to calculate fingerprints for windows
		// ending before the minimum chunk size.
		while ((chunkLength =
		inputStream.read(mChunkBuffer, /off=/ 0, /len=/ initialReadLength))
		!= -1) {
		int b;
		long fingerprint = 0L;

		while ((b = inputStream.read()) != -1) {
		byte inByte = (byte) b;
		byte outByte = getCurrentWindowStartByte(chunkLength);
		mChunkBuffer[chunkLength++] = inByte;

		fingerprint =
		mRabinFingerprint64.computeFingerprint64(inByte, outByte, fingerprint);

		if (chunkLength >= mMaxChunkSize
		\|\| (chunkLength >= mMinChunkSize
		&& mBreakpointPredicate.isBreakpoint(
		mFingerprintMixer.mix(fingerprint)))) {
		chunkConsumer.accept(Arrays.copyOf(mChunkBuffer, chunkLength));
		chunkLength = 0;
		break;
		}
		}

		if (chunkLength > 0) {
		chunkConsumer.accept(Arrays.copyOf(mChunkBuffer, chunkLength));
		}
		}
		}

		private byte getCurrentWindowStartByte(int chunkLength) {
		if (chunkLength < mMinChunkSize) {
		return DEFAULT_OUT_BYTE;
		} else {
		return mChunkBuffer[chunkLength - WINDOW_SIZE];
		}
		}

		/** Whether the current fingerprint indicates the end of a chunk. */
		public interface BreakpointPredicate {

		/**
		* Returns {@code true} if the fingerprint of the last {@code WINDOW_SIZE} bytes indicates
		* the chunk ought to end at this position.
		*
		* @param fingerprint Fingerprint of the last {@code WINDOW_SIZE} bytes.
		* @return Whether this ought to be a chunk breakpoint.
		*/
		boolean isBreakpoint(long fingerprint);
		}
		}

services/backup/java/com/android/server/backup/encryption/chunking/cdc/FingerprintMixer.java

0 → 100644

+95 −0

Original line number	Diff line number	Diff line
		/*
		* Copyright (C) 2018 The Android Open Source Project
		*
		* Licensed under the Apache License, Version 2.0 (the "License");
		* you may not use this file except in compliance with the License.
		* You may obtain a copy of the License at
		*
		* http://www.apache.org/licenses/LICENSE-2.0
		*
		* Unless required by applicable law or agreed to in writing, software
		* distributed under the License is distributed on an "AS IS" BASIS,
		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
		* See the License for the specific language governing permissions and
		* limitations under the License.
		*/

		package com.android.server.backup.encryption.chunking.cdc;

		import static com.android.internal.util.Preconditions.checkArgument;

		import java.nio.ByteBuffer;
		import java.nio.charset.StandardCharsets;
		import java.security.InvalidKeyException;

		import javax.crypto.SecretKey;

		/**
		* Helper for mixing fingerprint with key material.
		*
		* <p>We do this as otherwise the Rabin fingerprint leaks information about the plaintext. i.e., if
		* two users have the same file, it will be partitioned by Rabin in the same way, allowing us to
		* infer that it is the same as another user's file.
		*
		* <p>By mixing the fingerprint with the user's secret key, the chunking method is different on a
		* per key basis. Each application has its own {@link SecretKey}, so we cannot infer that a file is
		* the same even across multiple applications owned by the same user, never mind across multiple
		* users.
		*
		* <p>Instead of directly mixing the fingerprint with the user's secret, we first securely and
		* deterministically derive a secondary chunking key. As Rabin is not a cryptographically secure
		* hash, it might otherwise leak information about the user's secret. This prevents that from
		* happening.
		*/
		public class FingerprintMixer {
		public static final int SALT_LENGTH_BYTES = 256 / Byte.SIZE;
		private static final String DERIVED_KEY_NAME = "RabinFingerprint64Mixer";

		private final long mAddend;
		private final long mMultiplicand;

		/**
		* A new instance from a given secret key and salt. Salt must be the same across incremental
		* backups, or a different chunking strategy will be used each time, defeating the dedup.
		*
		* @param secretKey The application-specific secret.
		* @param salt The salt.
		* @throws InvalidKeyException If the encoded form of {@code secretKey} is inaccessible.
		*/
		public FingerprintMixer(SecretKey secretKey, byte[] salt) throws InvalidKeyException {
		checkArgument(salt.length == SALT_LENGTH_BYTES, "Requires a 256-bit salt.");
		byte[] keyBytes = secretKey.getEncoded();
		if (keyBytes == null) {
		throw new InvalidKeyException("SecretKey must support encoding for FingerprintMixer.");
		}
		byte[] derivedKey =
		Hkdf.hkdf(keyBytes, salt, DERIVED_KEY_NAME.getBytes(StandardCharsets.UTF_8));
		ByteBuffer buffer = ByteBuffer.wrap(derivedKey);
		mAddend = buffer.getLong();
		// Multiplicand must be odd - otherwise we lose some bits of the Rabin fingerprint when
		// mixing
		mMultiplicand = buffer.getLong() \| 1;
		}

		/**
		* Mixes the fingerprint with the derived key material. This is performed by adding part of the
		* derived key and multiplying by another part of the derived key (which is forced to be odd, so
		* that the operation is reversible).
		*
		* @param fingerprint A 64-bit Rabin fingerprint.
		* @return The mixed fingerprint.
		*/
		long mix(long fingerprint) {
		return ((fingerprint + mAddend) * mMultiplicand);
		}

		/** The addend part of the derived key. */
		long getAddend() {
		return mAddend;
		}

		/** The multiplicand part of the derived key. */
		long getMultiplicand() {
		return mMultiplicand;
		}
		}

services/backup/java/com/android/server/backup/encryption/chunking/cdc/Hkdf.java

0 → 100644

+115 −0

Original line number	Diff line number	Diff line
		/*
		* Copyright (C) 2018 The Android Open Source Project
		*
		* Licensed under the Apache License, Version 2.0 (the "License");
		* you may not use this file except in compliance with the License.
		* You may obtain a copy of the License at
		*
		* http://www.apache.org/licenses/LICENSE-2.0
		*
		* Unless required by applicable law or agreed to in writing, software
		* distributed under the License is distributed on an "AS IS" BASIS,
		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
		* See the License for the specific language governing permissions and
		* limitations under the License.
		*/

		package com.android.server.backup.encryption.chunking.cdc;

		import static com.android.internal.util.Preconditions.checkNotNull;

		import java.security.InvalidKeyException;
		import java.security.NoSuchAlgorithmException;

		import javax.crypto.Mac;
		import javax.crypto.spec.SecretKeySpec;

		/**
		* Secure HKDF utils. Allows client to deterministically derive additional key material from a base
		* secret. If the derived key material is compromised, this does not in of itself compromise the
		* root secret.
		*
		* <p>TODO(b/116575321): After all code is ported, rename this class to HkdfUtils.
		*/
		public final class Hkdf {
		private static final byte[] CONSTANT_01 = {0x01};
		private static final String HmacSHA256 = "HmacSHA256";
		private static final String AES = "AES";

		/**
		* Implements HKDF (RFC 5869) with the SHA-256 hash and a 256-bit output key length.
		*
		* <p>IMPORTANT: The use or edit of this method requires a security review.
		*
		* @param masterKey Master key from which to derive sub-keys.
		* @param salt A randomly generated 256-bit byte string.
		* @param data Arbitrary information that is bound to the derived key (i.e., used in its
		* creation).
		* @return Raw derived key bytes = HKDF-SHA256(masterKey, salt, data).
		* @throws InvalidKeyException If the salt can not be used as a valid key.
		*/
		static byte[] hkdf(byte[] masterKey, byte[] salt, byte[] data) throws InvalidKeyException {
		checkNotNull(masterKey, "HKDF requires master key to be set.");
		checkNotNull(salt, "HKDF requires a salt.");
		checkNotNull(data, "No data provided to HKDF.");
		return hkdfSha256Expand(hkdfSha256Extract(masterKey, salt), data);
		}

		private Hkdf() {}

		/**
		* The HKDF (RFC 5869) extraction function, using the SHA-256 hash function. This function is
		* used to pre-process the {@code inputKeyMaterial} and mix it with the {@code salt}, producing
		* output suitable for use with HKDF expansion function (which produces the actual derived key).
		*
		* <p>IMPORTANT: The use or edit of this method requires a security review.
		*
		* @see #hkdfSha256Expand(byte[], byte[])
		* @return HMAC-SHA256(salt, inputKeyMaterial) (salt is the "key" for the HMAC)
		* @throws InvalidKeyException If the salt can not be used as a valid key.
		*/
		private static byte[] hkdfSha256Extract(byte[] inputKeyMaterial, byte[] salt)
		throws InvalidKeyException {
		// Note that the SecretKey encoding format is defined to be RAW, so the encoded form should
		// be consistent across implementations.
		Mac sha256;
		try {
		sha256 = Mac.getInstance(HmacSHA256);
		} catch (NoSuchAlgorithmException e) {
		// This can not happen - HmacSHA256 is supported by the platform.
		throw new AssertionError(e);
		}
		sha256.init(new SecretKeySpec(salt, AES));

		return sha256.doFinal(inputKeyMaterial);
		}

		/**
		* Special case of HKDF (RFC 5869) expansion function, using the SHA-256 hash function and
		* allowing for a maximum output length of 256 bits.
		*
		* <p>IMPORTANT: The use or edit of this method requires a security review.
		*
		* @param pseudoRandomKey Generated by {@link #hkdfSha256Extract(byte[], byte[])}.
		* @param info Arbitrary information the derived key should be bound to.
		* @return Raw derived key bytes = HMAC-SHA256(pseudoRandomKey, info \| 0x01).
		* @throws InvalidKeyException If the salt can not be used as a valid key.
		*/
		private static byte[] hkdfSha256Expand(byte[] pseudoRandomKey, byte[] info)
		throws InvalidKeyException {
		// Note that RFC 5869 computes number of blocks N = ceil(hash length / output length), but
		// here we only deal with a 256 bit hash up to a 256 bit output, yielding N=1.
		Mac sha256;
		try {
		sha256 = Mac.getInstance(HmacSHA256);
		} catch (NoSuchAlgorithmException e) {
		// This can not happen - HmacSHA256 is supported by the platform.
		throw new AssertionError(e);
		}
		sha256.init(new SecretKeySpec(pseudoRandomKey, AES));

		sha256.update(info);
		sha256.update(CONSTANT_01);
		return sha256.doFinal();
		}
		}

services/backup/java/com/android/server/backup/encryption/chunking/cdc/IsChunkBreakpoint.java

0 → 100644

+78 −0

Original line number	Diff line number	Diff line
		/*
		* Copyright (C) 2018 The Android Open Source Project
		*
		* Licensed under the Apache License, Version 2.0 (the "License");
		* you may not use this file except in compliance with the License.
		* You may obtain a copy of the License at
		*
		* http://www.apache.org/licenses/LICENSE-2.0
		*
		* Unless required by applicable law or agreed to in writing, software
		* distributed under the License is distributed on an "AS IS" BASIS,
		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
		* See the License for the specific language governing permissions and
		* limitations under the License.
		*/

		package com.android.server.backup.encryption.chunking.cdc;

		import static com.android.internal.util.Preconditions.checkArgument;

		import com.android.server.backup.encryption.chunking.cdc.ContentDefinedChunker.BreakpointPredicate;

		/**
		* Function to determine whether a 64-bit fingerprint ought to be a chunk breakpoint.
		*
		* <p>This works by checking whether there are at least n leading zeros in the fingerprint. n is
		* calculated to on average cause a breakpoint after a given number of trials (provided in the
		* constructor). This allows us to choose a number of trials that gives a desired average chunk
		* size. This works because the fingerprint is pseudo-randomly distributed.
		*/
		public class IsChunkBreakpoint implements BreakpointPredicate {
		private final int mLeadingZeros;
		private final long mBitmask;

		/**
		* A new instance that causes a breakpoint after a given number of trials on average.
		*
		* @param averageNumberOfTrialsUntilBreakpoint The number of trials after which on average to
		* create a new chunk. If this is not a power of 2, some precision is sacrificed (i.e., on
		* average, breaks will actually happen after the nearest power of 2 to the average number
		* of trials passed in).
		*/
		public IsChunkBreakpoint(long averageNumberOfTrialsUntilBreakpoint) {
		checkArgument(
		averageNumberOfTrialsUntilBreakpoint >= 0,
		"Average number of trials must be non-negative");

		// Want n leading zeros after t trials.
		// P(leading zeros = n) = 1/2^n
		// Expected num trials to get n leading zeros = 1/2^-n
		// t = 1/2^-n
		// n = log2(t)
		mLeadingZeros = (int) Math.round(log2(averageNumberOfTrialsUntilBreakpoint));
		mBitmask = ~(~0L >>> mLeadingZeros);
		}

		/**
		* Returns {@code true} if {@code fingerprint} indicates that there should be a chunk
		* breakpoint.
		*/
		@Override
		public boolean isBreakpoint(long fingerprint) {
		return (fingerprint & mBitmask) == 0;
		}

		/** Returns the number of leading zeros in the fingerprint that causes a breakpoint. */
		public int getLeadingZeros() {
		return mLeadingZeros;
		}

		/**
		* Calculates log base 2 of x. Not the most efficient possible implementation, but it's simple,
		* obviously correct, and is only invoked on object construction.
		*/
		private static double log2(double x) {
		return Math.log(x) / Math.log(2);
		}
		}

services/backup/java/com/android/server/backup/encryption/chunking/cdc/RabinFingerprint64.java

0 → 100644

+113 −0

Original line number	Diff line number	Diff line
		/*
		* Copyright (C) 2018 The Android Open Source Project
		*
		* Licensed under the Apache License, Version 2.0 (the "License");
		* you may not use this file except in compliance with the License.
		* You may obtain a copy of the License at
		*
		* http://www.apache.org/licenses/LICENSE-2.0
		*
		* Unless required by applicable law or agreed to in writing, software
		* distributed under the License is distributed on an "AS IS" BASIS,
		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
		* See the License for the specific language governing permissions and
		* limitations under the License.
		*/

		package com.android.server.backup.encryption.chunking.cdc;

		/** Helper to calculate a 64-bit Rabin fingerprint over a 31-byte window. */
		public class RabinFingerprint64 {
		private static final long DEFAULT_IRREDUCIBLE_POLYNOMIAL_64 = 0x000000000000001BL;
		private static final int POLYNOMIAL_DEGREE = 64;
		private static final int SLIDING_WINDOW_SIZE_BYTES = 31;

		private final long mPoly64;
		// Auxiliary tables to speed up the computation of Rabin fingerprints.
		private final long[] mTableFP64 = new long[256];
		private final long[] mTableOutByte = new long[256];

		/**
		* Constructs a new instance over the given irreducible 64-degree polynomial. It is up to the
		* caller to determine that the polynomial is irreducible. If it is not the fingerprinting will
		* not behave as expected.
		*
		* @param poly64 The polynomial.
		*/
		public RabinFingerprint64(long poly64) {
		mPoly64 = poly64;
		}

		/** Constructs a new instance using {@code x^64 + x^4 + x + 1} as the irreducible polynomial. */
		public RabinFingerprint64() {
		this(DEFAULT_IRREDUCIBLE_POLYNOMIAL_64);
		computeFingerprintTables64();
		computeFingerprintTables64Windowed();
		}

		/**
		* Computes the fingerprint for the new sliding window given the fingerprint of the previous
		* sliding window, the byte sliding in, and the byte sliding out.
		*
		* @param inChar The new char coming into the sliding window.
		* @param outChar The left most char sliding out of the window.
		* @param fingerPrint Fingerprint for previous window.
		* @return New fingerprint for the new sliding window.
		*/
		public long computeFingerprint64(byte inChar, byte outChar, long fingerPrint) {
		return (fingerPrint << 8)
		^ (inChar & 0xFF)
		^ mTableFP64[(int) (fingerPrint >>> 56)]
		^ mTableOutByte[outChar & 0xFF];
		}

		/** Compute auxiliary tables to speed up the fingerprint computation. */
		private void computeFingerprintTables64() {
		long[] degreesRes64 = new long[POLYNOMIAL_DEGREE];
		degreesRes64[0] = mPoly64;
		for (int i = 1; i < POLYNOMIAL_DEGREE; i++) {
		if ((degreesRes64[i - 1] & (1L << 63)) == 0) {
		degreesRes64[i] = degreesRes64[i - 1] << 1;
		} else {
		degreesRes64[i] = (degreesRes64[i - 1] << 1) ^ mPoly64;
		}
		}
		for (int i = 0; i < 256; i++) {
		int currIndex = i;
		for (int j = 0; (currIndex > 0) && (j < 8); j++) {
		if ((currIndex & 0x1) == 1) {
		mTableFP64[i] ^= degreesRes64[j];
		}
		currIndex >>>= 1;
		}
		}
		}

		/**
		* Compute auxiliary table {@code mTableOutByte} to facilitate the computing of fingerprints for
		* sliding windows. This table is to take care of the effect on the fingerprint when the
		* leftmost byte in the window slides out.
		*/
		private void computeFingerprintTables64Windowed() {
		// Auxiliary array degsRes64[8] defined by: <code>degsRes64[i] = x^(8 *
		// SLIDING_WINDOW_SIZE_BYTES + i) mod this.mPoly64.</code>
		long[] degsRes64 = new long[8];
		degsRes64[0] = mPoly64;
		for (int i = 65; i < 8 * (SLIDING_WINDOW_SIZE_BYTES + 1); i++) {
		if ((degsRes64[(i - 1) % 8] & (1L << 63)) == 0) {
		degsRes64[i % 8] = degsRes64[(i - 1) % 8] << 1;
		} else {
		degsRes64[i % 8] = (degsRes64[(i - 1) % 8] << 1) ^ mPoly64;
		}
		}
		for (int i = 0; i < 256; i++) {
		int currIndex = i;
		for (int j = 0; (currIndex > 0) && (j < 8); j++) {
		if ((currIndex & 0x1) == 1) {
		mTableOutByte[i] ^= degsRes64[j];
		}
		currIndex >>>= 1;
		}
		}
		}
		}