Merge "Import BackupFileBuilder" (295aaad6) · Commits · e / os / android_frameworks_base

packages/BackupEncryption/src/com/android/server/backup/encryption/chunking/BackupFileBuilder.java

0 → 100644

+232 −0

Original line number	Diff line number	Diff line
		/*
		* Copyright (C) 2019 The Android Open Source Project
		*
		* Licensed under the Apache License, Version 2.0 (the "License");
		* you may not use this file except in compliance with the License.
		* You may obtain a copy of the License at
		*
		* http://www.apache.org/licenses/LICENSE-2.0
		*
		* Unless required by applicable law or agreed to in writing, software
		* distributed under the License is distributed on an "AS IS" BASIS,
		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
		* See the License for the specific language governing permissions and
		* limitations under the License.
		*/

		package com.android.server.backup.encryption.chunking;

		import static com.android.internal.util.Preconditions.checkArgument;
		import static com.android.internal.util.Preconditions.checkNotNull;
		import static com.android.internal.util.Preconditions.checkState;

		import android.annotation.Nullable;
		import android.util.Slog;

		import com.android.server.backup.encryption.chunk.ChunkHash;
		import com.android.server.backup.encryption.chunk.ChunkListingMap;
		import com.android.server.backup.encryption.protos.nano.ChunksMetadataProto;

		import java.io.IOException;
		import java.io.OutputStream;
		import java.util.ArrayList;
		import java.util.Arrays;
		import java.util.Collections;
		import java.util.HashMap;
		import java.util.List;
		import java.util.Map;

		/**
		* Writes batches of {@link EncryptedChunk} to a diff script, and generates the associated {@link
		* ChunksMetadataProto.ChunkListing} and {@link ChunksMetadataProto.ChunkOrdering}.
		*/
		public class BackupFileBuilder {
		private static final String TAG = "BackupFileBuilder";

		private static final int BYTES_PER_KILOBYTE = 1024;

		private final BackupWriter mBackupWriter;
		private final EncryptedChunkEncoder mEncryptedChunkEncoder;
		private final ChunkListingMap mOldChunkListing;
		private final ChunksMetadataProto.ChunkListing mNewChunkListing;
		private final ChunksMetadataProto.ChunkOrdering mChunkOrdering;
		private final List<ChunksMetadataProto.Chunk> mKnownChunks = new ArrayList<>();
		private final List<Integer> mKnownStarts = new ArrayList<>();
		private final Map<ChunkHash, Long> mChunkStartPositions;

		private long mNewChunksSizeBytes;
		private boolean mFinished;

		/**
		* Constructs a new instance which writes raw data to the given {@link OutputStream}, without
		* generating a diff.
		*
		* <p>This class never closes the output stream.
		*/
		public static BackupFileBuilder createForNonIncremental(OutputStream outputStream) {
		return new BackupFileBuilder(
		new RawBackupWriter(outputStream), new ChunksMetadataProto.ChunkListing());
		}

		/**
		* Constructs a new instance which writes a diff script to the given {@link OutputStream} using
		* a {@link SingleStreamDiffScriptWriter}.
		*
		* <p>This class never closes the output stream.
		*
		* @param oldChunkListing against which the diff will be generated.
		*/
		public static BackupFileBuilder createForIncremental(
		OutputStream outputStream, ChunksMetadataProto.ChunkListing oldChunkListing) {
		return new BackupFileBuilder(
		DiffScriptBackupWriter.newInstance(outputStream), oldChunkListing);
		}

		private BackupFileBuilder(
		BackupWriter backupWriter, ChunksMetadataProto.ChunkListing oldChunkListing) {
		this.mBackupWriter = backupWriter;
		// TODO(b/77188289): Use InlineLengthsEncryptedChunkEncoder for key-value backups
		this.mEncryptedChunkEncoder = new LengthlessEncryptedChunkEncoder();
		this.mOldChunkListing = ChunkListingMap.fromProto(oldChunkListing);

		mNewChunkListing = new ChunksMetadataProto.ChunkListing();
		mNewChunkListing.cipherType = ChunksMetadataProto.AES_256_GCM;
		mNewChunkListing.chunkOrderingType = ChunksMetadataProto.CHUNK_ORDERING_TYPE_UNSPECIFIED;

		mChunkOrdering = new ChunksMetadataProto.ChunkOrdering();
		mChunkStartPositions = new HashMap<>();
		}

		/**
		* Writes the given chunks to the output stream, and adds them to the new chunk listing and
		* chunk ordering.
		*
		* <p>Sorts the chunks in lexicographical order before writing.
		*
		* @param allChunks The hashes of all the chunks, in the order they appear in the plaintext.
		* @param newChunks A map from hash to {@link EncryptedChunk} containing the new chunks not
		* present in the previous backup.
		*/
		public void writeChunks(List<ChunkHash> allChunks, Map<ChunkHash, EncryptedChunk> newChunks)
		throws IOException {
		checkState(!mFinished, "Cannot write chunks after flushing.");

		List<ChunkHash> sortedChunks = new ArrayList<>(allChunks);
		Collections.sort(sortedChunks);
		for (ChunkHash chunkHash : sortedChunks) {
		// As we have already included this chunk in the backup file, don't add it again to
		// deduplicate identical chunks.
		if (!mChunkStartPositions.containsKey(chunkHash)) {
		// getBytesWritten() gives us the start of the chunk.
		mChunkStartPositions.put(chunkHash, mBackupWriter.getBytesWritten());

		writeChunkToFileAndListing(chunkHash, newChunks);
		}
		}

		long totalSizeKb = mBackupWriter.getBytesWritten() / BYTES_PER_KILOBYTE;
		long newChunksSizeKb = mNewChunksSizeBytes / BYTES_PER_KILOBYTE;
		Slog.d(
		TAG,
		"Total backup size: "
		+ totalSizeKb
		+ " kb, new chunks size: "
		+ newChunksSizeKb
		+ " kb");

		for (ChunkHash chunkHash : allChunks) {
		mKnownStarts.add(mChunkStartPositions.get(chunkHash).intValue());
		}
		}

		/**
		* Returns a new listing for all of the chunks written so far, setting the given fingerprint
		* mixer salt (this overrides the {@link ChunksMetadataProto.ChunkListing#fingerprintMixerSalt}
		* in the old {@link ChunksMetadataProto.ChunkListing} passed into the
		* {@link #BackupFileBuilder).
		*/
		public ChunksMetadataProto.ChunkListing getNewChunkListing(
		@Nullable byte[] fingerprintMixerSalt) {
		// TODO: b/141537803 Add check to ensure this is called only once per instance
		mNewChunkListing.fingerprintMixerSalt =
		fingerprintMixerSalt != null
		? Arrays.copyOf(fingerprintMixerSalt, fingerprintMixerSalt.length)
		: new byte[0];
		mNewChunkListing.chunks = mKnownChunks.toArray(new ChunksMetadataProto.Chunk[0]);
		return mNewChunkListing;
		}

		/** Returns a new ordering for all of the chunks written so far, setting the given checksum. */
		public ChunksMetadataProto.ChunkOrdering getNewChunkOrdering(byte[] checksum) {
		// TODO: b/141537803 Add check to ensure this is called only once per instance
		mChunkOrdering.starts = new int[mKnownStarts.size()];
		for (int i = 0; i < mKnownStarts.size(); i++) {
		mChunkOrdering.starts[i] = mKnownStarts.get(i).intValue();
		}
		mChunkOrdering.checksum = Arrays.copyOf(checksum, checksum.length);
		return mChunkOrdering;
		}

		/**
		* Finishes the backup file by writing the chunk metadata and metadata position.
		*
		* <p>Once this is called, calling {@link #writeChunks(List, Map)} will throw {@link
		* IllegalStateException}.
		*/
		public void finish(ChunksMetadataProto.ChunksMetadata metadata) throws IOException {
		checkNotNull(metadata, "Metadata cannot be null");

		long startOfMetadata = mBackupWriter.getBytesWritten();
		mBackupWriter.writeBytes(ChunksMetadataProto.ChunksMetadata.toByteArray(metadata));
		mBackupWriter.writeBytes(toByteArray(startOfMetadata));

		mBackupWriter.flush();
		mFinished = true;
		}

		/**
		* Checks if the given chunk hash references an existing chunk or a new chunk, and adds this
		* chunk to the backup file and new chunk listing.
		*/
		private void writeChunkToFileAndListing(
		ChunkHash chunkHash, Map<ChunkHash, EncryptedChunk> newChunks) throws IOException {
		checkNotNull(chunkHash, "Hash cannot be null");

		if (mOldChunkListing.hasChunk(chunkHash)) {
		ChunkListingMap.Entry oldChunk = mOldChunkListing.getChunkEntry(chunkHash);
		mBackupWriter.writeChunk(oldChunk.getStart(), oldChunk.getLength());

		checkArgument(oldChunk.getLength() >= 0, "Chunk must have zero or positive length");
		addChunk(chunkHash.getHash(), oldChunk.getLength());
		} else if (newChunks.containsKey(chunkHash)) {
		EncryptedChunk newChunk = newChunks.get(chunkHash);
		mEncryptedChunkEncoder.writeChunkToWriter(mBackupWriter, newChunk);
		int length = mEncryptedChunkEncoder.getEncodedLengthOfChunk(newChunk);
		mNewChunksSizeBytes += length;

		checkArgument(length >= 0, "Chunk must have zero or positive length");
		addChunk(chunkHash.getHash(), length);
		} else {
		throw new IllegalArgumentException(
		"Chunk did not exist in old chunks or new chunks: " + chunkHash);
		}
		}

		private void addChunk(byte[] chunkHash, int length) {
		ChunksMetadataProto.Chunk chunk = new ChunksMetadataProto.Chunk();
		chunk.hash = Arrays.copyOf(chunkHash, chunkHash.length);
		chunk.length = length;
		mKnownChunks.add(chunk);
		}

		private static byte[] toByteArray(long value) {
		// Note that this code needs to stay compatible with GWT, which has known
		// bugs when narrowing byte casts of long values occur.
		byte[] result = new byte[8];
		for (int i = 7; i >= 0; i--) {
		result[i] = (byte) (value & 0xffL);
		value >>= 8;
		}
		return result;
		}
		}

packages/BackupEncryption/test/robolectric/src/com/android/server/backup/encryption/chunking/BackupFileBuilderTest.java

0 → 100644

+614 −0

File added.

Preview size limit exceeded, changes collapsed.

packages/BackupEncryption/test/robolectric/src/com/android/server/backup/encryption/testing/DiffScriptProcessor.java

0 → 100644

+256 −0

Original line number	Diff line number	Diff line
		/*
		* Copyright (C) 2019 The Android Open Source Project
		*
		* Licensed under the Apache License, Version 2.0 (the "License");
		* you may not use this file except in compliance with the License.
		* You may obtain a copy of the License at
		*
		* http://www.apache.org/licenses/LICENSE-2.0
		*
		* Unless required by applicable law or agreed to in writing, software
		* distributed under the License is distributed on an "AS IS" BASIS,
		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
		* See the License for the specific language governing permissions and
		* limitations under the License.
		*/

		package com.android.server.backup.encryption.testing;

		import static com.android.internal.util.Preconditions.checkArgument;
		import static com.android.internal.util.Preconditions.checkNotNull;

		import static java.nio.charset.StandardCharsets.UTF_8;

		import java.io.ByteArrayOutputStream;
		import java.io.File;
		import java.io.FileOutputStream;
		import java.io.IOException;
		import java.io.InputStream;
		import java.io.OutputStream;
		import java.io.RandomAccessFile;
		import java.util.Locale;
		import java.util.Optional;
		import java.util.Scanner;
		import java.util.regex.Pattern;

		/**
		* To be used as part of a fake backup server. Processes a Scotty diff script.
		*
		* <p>A Scotty diff script consists of an ASCII line denoting a command, optionally followed by a
		* range of bytes. Command format is either
		*
		* <ul>
		* <li>A single 64-bit integer, followed by a new line: this denotes that the given number of
		* bytes are to follow in the stream. These bytes should be written directly to the new file.
		* <li>Two 64-bit integers, separated by a hyphen, followed by a new line: this says that the
		* given range of bytes from the original file ought to be copied into the new file.
		* </ul>
		*/
		public class DiffScriptProcessor {

		private static final int COPY_BUFFER_SIZE = 1024;

		private static final String READ_MODE = "r";
		private static final Pattern VALID_COMMAND_PATTERN = Pattern.compile("^\\d+(-\\d+)?$");

		private final File mInput;
		private final File mOutput;
		private final long mInputLength;

		/**
		* A new instance, with {@code input} as previous file, and {@code output} as new file.
		*
		* @param input Previous file from which ranges of bytes are to be copied. This file should be
		* immutable.
		* @param output Output file, to which the new data should be written.
		* @throws IllegalArgumentException if input does not exist.
		*/
		public DiffScriptProcessor(File input, File output) {
		checkArgument(input.exists(), "input file did not exist.");
		mInput = input;
		mInputLength = input.length();
		mOutput = checkNotNull(output);
		}

		public void process(InputStream diffScript) throws IOException, MalformedDiffScriptException {
		RandomAccessFile randomAccessInput = new RandomAccessFile(mInput, READ_MODE);

		try (FileOutputStream outputStream = new FileOutputStream(mOutput)) {
		while (true) {
		Optional<String> commandString = readCommand(diffScript);
		if (!commandString.isPresent()) {
		return;
		}
		Command command = Command.parse(commandString.get());

		if (command.mIsRange) {
		checkFileRange(command.mCount, command.mLimit);
		copyRange(randomAccessInput, outputStream, command.mCount, command.mLimit);
		} else {
		long bytesCopied = copyBytes(diffScript, outputStream, command.mCount);
		if (bytesCopied < command.mCount) {
		throw new MalformedDiffScriptException(
		String.format(
		Locale.US,
		"Command to copy %d bytes from diff script, but only %d"
		+ " bytes available",
		command.mCount,
		bytesCopied));
		}
		if (diffScript.read() != '\n') {
		throw new MalformedDiffScriptException("Expected new line after bytes.");
		}
		}
		}
		}
		}

		private void checkFileRange(long start, long end) throws MalformedDiffScriptException {
		if (end < start) {
		throw new MalformedDiffScriptException(
		String.format(
		Locale.US,
		"Command to copy %d-%d bytes from original file, but %2$d < %1$d.",
		start,
		end));
		}

		if (end >= mInputLength) {
		throw new MalformedDiffScriptException(
		String.format(
		Locale.US,
		"Command to copy %d-%d bytes from original file, but file is only %d"
		+ " bytes long.",
		start,
		end,
		mInputLength));
		}
		}

		/**
		* Reads a command from the input stream.
		*
		* @param inputStream The input.
		* @return Optional of command, or empty if EOF.
		*/
		private static Optional<String> readCommand(InputStream inputStream) throws IOException {
		ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();

		int b;
		while (!isEndOfCommand(b = inputStream.read())) {
		byteArrayOutputStream.write(b);
		}

		byte[] bytes = byteArrayOutputStream.toByteArray();
		if (bytes.length == 0) {
		return Optional.empty();
		} else {
		return Optional.of(new String(bytes, UTF_8));
		}
		}

		/**
		* If the given output from {@link InputStream#read()} is the end of a command - i.e., a new
		* line or the EOF.
		*
		* @param b The byte or -1.
		* @return {@code true} if ends the command.
		*/
		private static boolean isEndOfCommand(int b) {
		return b == -1 \|\| b == '\n';
		}

		/**
		* Copies {@code n} bytes from {@code inputStream} to {@code outputStream}.
		*
		* @return The number of bytes copied.
		* @throws IOException if there was a problem reading or writing.
		*/
		private static long copyBytes(InputStream inputStream, OutputStream outputStream, long n)
		throws IOException {
		byte[] buffer = new byte[COPY_BUFFER_SIZE];
		long copied = 0;
		while (n - copied > COPY_BUFFER_SIZE) {
		long read = copyBlock(inputStream, outputStream, buffer, COPY_BUFFER_SIZE);
		if (read <= 0) {
		return copied;
		}
		}
		while (n - copied > 0) {
		copied += copyBlock(inputStream, outputStream, buffer, (int) (n - copied));
		}
		return copied;
		}

		private static long copyBlock(
		InputStream inputStream, OutputStream outputStream, byte[] buffer, int size)
		throws IOException {
		int read = inputStream.read(buffer, 0, size);
		outputStream.write(buffer, 0, read);
		return read;
		}

		/**
		* Copies the given range of bytes from the input file to the output stream.
		*
		* @param input The input file.
		* @param output The output stream.
		* @param start Start position in the input file.
		* @param end End position in the output file (inclusive).
		* @throws IOException if there was a problem reading or writing.
		*/
		private static void copyRange(RandomAccessFile input, OutputStream output, long start, long end)
		throws IOException {
		input.seek(start);

		// Inefficient but obviously correct. If tests become slow, optimize.
		for (; start <= end; start++) {
		output.write(input.read());
		}
		}

		/** Error thrown for a malformed diff script. */
		public static class MalformedDiffScriptException extends Exception {
		public MalformedDiffScriptException(String message) {
		super(message);
		}
		}

		/**
		* A command telling the processor either to insert n bytes, which follow, or copy n-m bytes
		* from the original file.
		*/
		private static class Command {
		private final long mCount;
		private final long mLimit;
		private final boolean mIsRange;

		private Command(long count, long limit, boolean isRange) {
		mCount = count;
		mLimit = limit;
		mIsRange = isRange;
		}

		/**
		* Attempts to parse the command string into a usable structure.
		*
		* @param command The command string, without a new line at the end.
		* @throws MalformedDiffScriptException if the command is not a valid diff script command.
		* @return The parsed command.
		*/
		private static Command parse(String command) throws MalformedDiffScriptException {
		if (!VALID_COMMAND_PATTERN.matcher(command).matches()) {
		throw new MalformedDiffScriptException("Bad command: " + command);
		}

		Scanner commandScanner = new Scanner(command);
		commandScanner.useDelimiter("-");
		long n = commandScanner.nextLong();
		if (!commandScanner.hasNextLong()) {
		return new Command(n, 0L, /isRange=/ false);
		}
		long m = commandScanner.nextLong();
		return new Command(n, m, /isRange=/ true);
		}
		}
		}

packages/BackupEncryption/test/robolectric/src/com/android/server/testing/CryptoTestUtils.java

+15 −0

Original line number	Diff line number	Diff line
		@@ -16,7 +16,11 @@

		package com.android.server.backup.testing;

		import com.android.server.backup.encryption.chunk.ChunkHash;
		import com.android.server.backup.encryption.protos.nano.ChunksMetadataProto;

		import java.security.NoSuchAlgorithmException;
		import java.util.Arrays;
		import java.util.Random;

		import javax.crypto.KeyGenerator;
		@@ -42,4 +46,15 @@ public class CryptoTestUtils {
		random.nextBytes(bytes);
		return bytes;
		}

		public static ChunksMetadataProto.Chunk newChunk(ChunkHash hash, int length) {
		return newChunk(hash.getHash(), length);
		}

		public static ChunksMetadataProto.Chunk newChunk(byte[] hash, int length) {
		ChunksMetadataProto.Chunk newChunk = new ChunksMetadataProto.Chunk();
		newChunk.hash = Arrays.copyOf(hash, hash.length);
		newChunk.length = length;
		return newChunk;
		}
		}