Import BackupFileBuilder (73bec2fc) · Commits · e / os / android_frameworks_base

packages/BackupEncryption/src/com/android/server/backup/encryption/chunking/BackupFileBuilder.java

0 → 100644

+232 −0

Original line number	Original line	Diff line number	Diff line
			/*
			* Copyright (C) 2019 The Android Open Source Project
			*
			* Licensed under the Apache License, Version 2.0 (the "License");
			* you may not use this file except in compliance with the License.
			* You may obtain a copy of the License at
			*
			* http://www.apache.org/licenses/LICENSE-2.0
			*
			* Unless required by applicable law or agreed to in writing, software
			* distributed under the License is distributed on an "AS IS" BASIS,
			* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
			* See the License for the specific language governing permissions and
			* limitations under the License.
			*/

			package com.android.server.backup.encryption.chunking;

			import static com.android.internal.util.Preconditions.checkArgument;
			import static com.android.internal.util.Preconditions.checkNotNull;
			import static com.android.internal.util.Preconditions.checkState;

			import android.annotation.Nullable;
			import android.util.Slog;

			import com.android.server.backup.encryption.chunk.ChunkHash;
			import com.android.server.backup.encryption.chunk.ChunkListingMap;
			import com.android.server.backup.encryption.protos.nano.ChunksMetadataProto;

			import java.io.IOException;
			import java.io.OutputStream;
			import java.util.ArrayList;
			import java.util.Arrays;
			import java.util.Collections;
			import java.util.HashMap;
			import java.util.List;
			import java.util.Map;

			/**
			* Writes batches of {@link EncryptedChunk} to a diff script, and generates the associated {@link
			* ChunksMetadataProto.ChunkListing} and {@link ChunksMetadataProto.ChunkOrdering}.
			*/
			public class BackupFileBuilder {
			private static final String TAG = "BackupFileBuilder";

			private static final int BYTES_PER_KILOBYTE = 1024;

			private final BackupWriter mBackupWriter;
			private final EncryptedChunkEncoder mEncryptedChunkEncoder;
			private final ChunkListingMap mOldChunkListing;
			private final ChunksMetadataProto.ChunkListing mNewChunkListing;
			private final ChunksMetadataProto.ChunkOrdering mChunkOrdering;
			private final List<ChunksMetadataProto.Chunk> mKnownChunks = new ArrayList<>();
			private final List<Integer> mKnownStarts = new ArrayList<>();
			private final Map<ChunkHash, Long> mChunkStartPositions;

			private long mNewChunksSizeBytes;
			private boolean mFinished;

			/**
			* Constructs a new instance which writes raw data to the given {@link OutputStream}, without
			* generating a diff.
			*
			* <p>This class never closes the output stream.
			*/
			public static BackupFileBuilder createForNonIncremental(OutputStream outputStream) {
			return new BackupFileBuilder(
			new RawBackupWriter(outputStream), new ChunksMetadataProto.ChunkListing());
			}

			/**
			* Constructs a new instance which writes a diff script to the given {@link OutputStream} using
			* a {@link SingleStreamDiffScriptWriter}.
			*
			* <p>This class never closes the output stream.
			*
			* @param oldChunkListing against which the diff will be generated.
			*/
			public static BackupFileBuilder createForIncremental(
			OutputStream outputStream, ChunksMetadataProto.ChunkListing oldChunkListing) {
			return new BackupFileBuilder(
			DiffScriptBackupWriter.newInstance(outputStream), oldChunkListing);
			}

			private BackupFileBuilder(
			BackupWriter backupWriter, ChunksMetadataProto.ChunkListing oldChunkListing) {
			this.mBackupWriter = backupWriter;
			// TODO(b/77188289): Use InlineLengthsEncryptedChunkEncoder for key-value backups
			this.mEncryptedChunkEncoder = new LengthlessEncryptedChunkEncoder();
			this.mOldChunkListing = ChunkListingMap.fromProto(oldChunkListing);

			mNewChunkListing = new ChunksMetadataProto.ChunkListing();
			mNewChunkListing.cipherType = ChunksMetadataProto.AES_256_GCM;
			mNewChunkListing.chunkOrderingType = ChunksMetadataProto.CHUNK_ORDERING_TYPE_UNSPECIFIED;

			mChunkOrdering = new ChunksMetadataProto.ChunkOrdering();
			mChunkStartPositions = new HashMap<>();
			}

			/**
			* Writes the given chunks to the output stream, and adds them to the new chunk listing and
			* chunk ordering.
			*
			* <p>Sorts the chunks in lexicographical order before writing.
			*
			* @param allChunks The hashes of all the chunks, in the order they appear in the plaintext.
			* @param newChunks A map from hash to {@link EncryptedChunk} containing the new chunks not
			* present in the previous backup.
			*/
			public void writeChunks(List<ChunkHash> allChunks, Map<ChunkHash, EncryptedChunk> newChunks)
			throws IOException {
			checkState(!mFinished, "Cannot write chunks after flushing.");

			List<ChunkHash> sortedChunks = new ArrayList<>(allChunks);
			Collections.sort(sortedChunks);
			for (ChunkHash chunkHash : sortedChunks) {
			// As we have already included this chunk in the backup file, don't add it again to
			// deduplicate identical chunks.
			if (!mChunkStartPositions.containsKey(chunkHash)) {
			// getBytesWritten() gives us the start of the chunk.
			mChunkStartPositions.put(chunkHash, mBackupWriter.getBytesWritten());

			writeChunkToFileAndListing(chunkHash, newChunks);
			}
			}

			long totalSizeKb = mBackupWriter.getBytesWritten() / BYTES_PER_KILOBYTE;
			long newChunksSizeKb = mNewChunksSizeBytes / BYTES_PER_KILOBYTE;
			Slog.d(
			TAG,
			"Total backup size: "
			+ totalSizeKb
			+ " kb, new chunks size: "
			+ newChunksSizeKb
			+ " kb");

			for (ChunkHash chunkHash : allChunks) {
			mKnownStarts.add(mChunkStartPositions.get(chunkHash).intValue());
			}
			}

			/**
			* Returns a new listing for all of the chunks written so far, setting the given fingerprint
			* mixer salt (this overrides the {@link ChunksMetadataProto.ChunkListing#fingerprintMixerSalt}
			* in the old {@link ChunksMetadataProto.ChunkListing} passed into the
			* {@link #BackupFileBuilder).
			*/
			public ChunksMetadataProto.ChunkListing getNewChunkListing(
			@Nullable byte[] fingerprintMixerSalt) {
			// TODO: b/141537803 Add check to ensure this is called only once per instance
			mNewChunkListing.fingerprintMixerSalt =
			fingerprintMixerSalt != null
			? Arrays.copyOf(fingerprintMixerSalt, fingerprintMixerSalt.length)
			: new byte[0];
			mNewChunkListing.chunks = mKnownChunks.toArray(new ChunksMetadataProto.Chunk[0]);
			return mNewChunkListing;
			}

			/** Returns a new ordering for all of the chunks written so far, setting the given checksum. */
			public ChunksMetadataProto.ChunkOrdering getNewChunkOrdering(byte[] checksum) {
			// TODO: b/141537803 Add check to ensure this is called only once per instance
			mChunkOrdering.starts = new int[mKnownStarts.size()];
			for (int i = 0; i < mKnownStarts.size(); i++) {
			mChunkOrdering.starts[i] = mKnownStarts.get(i).intValue();
			}
			mChunkOrdering.checksum = Arrays.copyOf(checksum, checksum.length);
			return mChunkOrdering;
			}

			/**
			* Finishes the backup file by writing the chunk metadata and metadata position.
			*
			* <p>Once this is called, calling {@link #writeChunks(List, Map)} will throw {@link
			* IllegalStateException}.
			*/
			public void finish(ChunksMetadataProto.ChunksMetadata metadata) throws IOException {
			checkNotNull(metadata, "Metadata cannot be null");

			long startOfMetadata = mBackupWriter.getBytesWritten();
			mBackupWriter.writeBytes(ChunksMetadataProto.ChunksMetadata.toByteArray(metadata));
			mBackupWriter.writeBytes(toByteArray(startOfMetadata));

			mBackupWriter.flush();
			mFinished = true;
			}

			/**
			* Checks if the given chunk hash references an existing chunk or a new chunk, and adds this
			* chunk to the backup file and new chunk listing.
			*/
			private void writeChunkToFileAndListing(
			ChunkHash chunkHash, Map<ChunkHash, EncryptedChunk> newChunks) throws IOException {
			checkNotNull(chunkHash, "Hash cannot be null");

			if (mOldChunkListing.hasChunk(chunkHash)) {
			ChunkListingMap.Entry oldChunk = mOldChunkListing.getChunkEntry(chunkHash);
			mBackupWriter.writeChunk(oldChunk.getStart(), oldChunk.getLength());

			checkArgument(oldChunk.getLength() >= 0, "Chunk must have zero or positive length");
			addChunk(chunkHash.getHash(), oldChunk.getLength());
			} else if (newChunks.containsKey(chunkHash)) {
			EncryptedChunk newChunk = newChunks.get(chunkHash);
			mEncryptedChunkEncoder.writeChunkToWriter(mBackupWriter, newChunk);
			int length = mEncryptedChunkEncoder.getEncodedLengthOfChunk(newChunk);
			mNewChunksSizeBytes += length;

			checkArgument(length >= 0, "Chunk must have zero or positive length");
			addChunk(chunkHash.getHash(), length);
			} else {
			throw new IllegalArgumentException(
			"Chunk did not exist in old chunks or new chunks: " + chunkHash);
			}
			}

			private void addChunk(byte[] chunkHash, int length) {
			ChunksMetadataProto.Chunk chunk = new ChunksMetadataProto.Chunk();
			chunk.hash = Arrays.copyOf(chunkHash, chunkHash.length);
			chunk.length = length;
			mKnownChunks.add(chunk);
			}

			private static byte[] toByteArray(long value) {
			// Note that this code needs to stay compatible with GWT, which has known
			// bugs when narrowing byte casts of long values occur.
			byte[] result = new byte[8];
			for (int i = 7; i >= 0; i--) {
			result[i] = (byte) (value & 0xffL);
			value >>= 8;
			}
			return result;
			}
			}

packages/BackupEncryption/test/robolectric/src/com/android/server/backup/encryption/chunking/BackupFileBuilderTest.java

0 → 100644

+614 −0

File added.

Preview size limit exceeded, changes collapsed.

packages/BackupEncryption/test/robolectric/src/com/android/server/backup/encryption/testing/DiffScriptProcessor.java

0 → 100644

+256 −0

Original line number	Original line	Diff line number	Diff line
			/*
			* Copyright (C) 2019 The Android Open Source Project
			*
			* Licensed under the Apache License, Version 2.0 (the "License");
			* you may not use this file except in compliance with the License.
			* You may obtain a copy of the License at
			*
			* http://www.apache.org/licenses/LICENSE-2.0
			*
			* Unless required by applicable law or agreed to in writing, software
			* distributed under the License is distributed on an "AS IS" BASIS,
			* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
			* See the License for the specific language governing permissions and
			* limitations under the License.
			*/

			package com.android.server.backup.encryption.testing;

			import static com.android.internal.util.Preconditions.checkArgument;
			import static com.android.internal.util.Preconditions.checkNotNull;

			import static java.nio.charset.StandardCharsets.UTF_8;

			import java.io.ByteArrayOutputStream;
			import java.io.File;
			import java.io.FileOutputStream;
			import java.io.IOException;
			import java.io.InputStream;
			import java.io.OutputStream;
			import java.io.RandomAccessFile;
			import java.util.Locale;
			import java.util.Optional;
			import java.util.Scanner;
			import java.util.regex.Pattern;

			/**
			* To be used as part of a fake backup server. Processes a Scotty diff script.
			*
			* <p>A Scotty diff script consists of an ASCII line denoting a command, optionally followed by a
			* range of bytes. Command format is either
			*
			* <ul>
			* <li>A single 64-bit integer, followed by a new line: this denotes that the given number of
			* bytes are to follow in the stream. These bytes should be written directly to the new file.
			* <li>Two 64-bit integers, separated by a hyphen, followed by a new line: this says that the
			* given range of bytes from the original file ought to be copied into the new file.
			* </ul>
			*/
			public class DiffScriptProcessor {

			private static final int COPY_BUFFER_SIZE = 1024;

			private static final String READ_MODE = "r";
			private static final Pattern VALID_COMMAND_PATTERN = Pattern.compile("^\\d+(-\\d+)?$");

			private final File mInput;
			private final File mOutput;
			private final long mInputLength;

			/**
			* A new instance, with {@code input} as previous file, and {@code output} as new file.
			*
			* @param input Previous file from which ranges of bytes are to be copied. This file should be
			* immutable.
			* @param output Output file, to which the new data should be written.
			* @throws IllegalArgumentException if input does not exist.
			*/
			public DiffScriptProcessor(File input, File output) {
			checkArgument(input.exists(), "input file did not exist.");
			mInput = input;
			mInputLength = input.length();
			mOutput = checkNotNull(output);
			}

			public void process(InputStream diffScript) throws IOException, MalformedDiffScriptException {
			RandomAccessFile randomAccessInput = new RandomAccessFile(mInput, READ_MODE);

			try (FileOutputStream outputStream = new FileOutputStream(mOutput)) {
			while (true) {
			Optional<String> commandString = readCommand(diffScript);
			if (!commandString.isPresent()) {
			return;
			}
			Command command = Command.parse(commandString.get());

			if (command.mIsRange) {
			checkFileRange(command.mCount, command.mLimit);
			copyRange(randomAccessInput, outputStream, command.mCount, command.mLimit);
			} else {
			long bytesCopied = copyBytes(diffScript, outputStream, command.mCount);
			if (bytesCopied < command.mCount) {
			throw new MalformedDiffScriptException(
			String.format(
			Locale.US,
			"Command to copy %d bytes from diff script, but only %d"
			+ " bytes available",
			command.mCount,
			bytesCopied));
			}
			if (diffScript.read() != '\n') {
			throw new MalformedDiffScriptException("Expected new line after bytes.");
			}
			}
			}
			}
			}

			private void checkFileRange(long start, long end) throws MalformedDiffScriptException {
			if (end < start) {
			throw new MalformedDiffScriptException(
			String.format(
			Locale.US,
			"Command to copy %d-%d bytes from original file, but %2$d < %1$d.",
			start,
			end));
			}

			if (end >= mInputLength) {
			throw new MalformedDiffScriptException(
			String.format(
			Locale.US,
			"Command to copy %d-%d bytes from original file, but file is only %d"
			+ " bytes long.",
			start,
			end,
			mInputLength));
			}
			}

			/**
			* Reads a command from the input stream.
			*
			* @param inputStream The input.
			* @return Optional of command, or empty if EOF.
			*/
			private static Optional<String> readCommand(InputStream inputStream) throws IOException {
			ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();

			int b;
			while (!isEndOfCommand(b = inputStream.read())) {
			byteArrayOutputStream.write(b);
			}

			byte[] bytes = byteArrayOutputStream.toByteArray();
			if (bytes.length == 0) {
			return Optional.empty();
			} else {
			return Optional.of(new String(bytes, UTF_8));
			}
			}

			/**
			* If the given output from {@link InputStream#read()} is the end of a command - i.e., a new
			* line or the EOF.
			*
			* @param b The byte or -1.
			* @return {@code true} if ends the command.
			*/
			private static boolean isEndOfCommand(int b) {
			return b == -1 \|\| b == '\n';
			}

			/**
			* Copies {@code n} bytes from {@code inputStream} to {@code outputStream}.
			*
			* @return The number of bytes copied.
			* @throws IOException if there was a problem reading or writing.
			*/
			private static long copyBytes(InputStream inputStream, OutputStream outputStream, long n)
			throws IOException {
			byte[] buffer = new byte[COPY_BUFFER_SIZE];
			long copied = 0;
			while (n - copied > COPY_BUFFER_SIZE) {
			long read = copyBlock(inputStream, outputStream, buffer, COPY_BUFFER_SIZE);
			if (read <= 0) {
			return copied;
			}
			}
			while (n - copied > 0) {
			copied += copyBlock(inputStream, outputStream, buffer, (int) (n - copied));
			}
			return copied;
			}

			private static long copyBlock(
			InputStream inputStream, OutputStream outputStream, byte[] buffer, int size)
			throws IOException {
			int read = inputStream.read(buffer, 0, size);
			outputStream.write(buffer, 0, read);
			return read;
			}

			/**
			* Copies the given range of bytes from the input file to the output stream.
			*
			* @param input The input file.
			* @param output The output stream.
			* @param start Start position in the input file.
			* @param end End position in the output file (inclusive).
			* @throws IOException if there was a problem reading or writing.
			*/
			private static void copyRange(RandomAccessFile input, OutputStream output, long start, long end)
			throws IOException {
			input.seek(start);

			// Inefficient but obviously correct. If tests become slow, optimize.
			for (; start <= end; start++) {
			output.write(input.read());
			}
			}

			/** Error thrown for a malformed diff script. */
			public static class MalformedDiffScriptException extends Exception {
			public MalformedDiffScriptException(String message) {
			super(message);
			}
			}

			/**
			* A command telling the processor either to insert n bytes, which follow, or copy n-m bytes
			* from the original file.
			*/
			private static class Command {
			private final long mCount;
			private final long mLimit;
			private final boolean mIsRange;

			private Command(long count, long limit, boolean isRange) {
			mCount = count;
			mLimit = limit;
			mIsRange = isRange;
			}

			/**
			* Attempts to parse the command string into a usable structure.
			*
			* @param command The command string, without a new line at the end.
			* @throws MalformedDiffScriptException if the command is not a valid diff script command.
			* @return The parsed command.
			*/
			private static Command parse(String command) throws MalformedDiffScriptException {
			if (!VALID_COMMAND_PATTERN.matcher(command).matches()) {
			throw new MalformedDiffScriptException("Bad command: " + command);
			}

			Scanner commandScanner = new Scanner(command);
			commandScanner.useDelimiter("-");
			long n = commandScanner.nextLong();
			if (!commandScanner.hasNextLong()) {
			return new Command(n, 0L, /isRange=/ false);
			}
			long m = commandScanner.nextLong();
			return new Command(n, m, /isRange=/ true);
			}
			}
			}

packages/BackupEncryption/test/robolectric/src/com/android/server/testing/CryptoTestUtils.java

+15 −0

Original line number	Original line	Diff line number	Diff line
	@@ -16,7 +16,11 @@

	package com.android.server.backup.testing;		package com.android.server.backup.testing;

			import com.android.server.backup.encryption.chunk.ChunkHash;
			import com.android.server.backup.encryption.protos.nano.ChunksMetadataProto;

	import java.security.NoSuchAlgorithmException;		import java.security.NoSuchAlgorithmException;
			import java.util.Arrays;
	import java.util.Random;		import java.util.Random;

	import javax.crypto.KeyGenerator;		import javax.crypto.KeyGenerator;
	@@ -42,4 +46,15 @@ public class CryptoTestUtils {
	random.nextBytes(bytes);		random.nextBytes(bytes);
	return bytes;		return bytes;
	}		}

			public static ChunksMetadataProto.Chunk newChunk(ChunkHash hash, int length) {
			return newChunk(hash.getHash(), length);
			}

			public static ChunksMetadataProto.Chunk newChunk(byte[] hash, int length) {
			ChunksMetadataProto.Chunk newChunk = new ChunksMetadataProto.Chunk();
			newChunk.hash = Arrays.copyOf(hash, hash.length);
			newChunk.length = length;
			return newChunk;
			}
	}		}