Merge "Ports over BackupStreamEncrypter and related classes."

/*

 * Copyright (C) 2019 The Android Open Source Project

 *

 * Licensed under the Apache License, Version 2.0 (the "License");

 * you may not use this file except in compliance with the License.

 * You may obtain a copy of the License at

 *

 *      http://www.apache.org/licenses/LICENSE-2.0

 *

 * Unless required by applicable law or agreed to in writing, software

 * distributed under the License is distributed on an "AS IS" BASIS,

 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

 * See the License for the specific language governing permissions and

 * limitations under the License.

 */

package com.android.server.backup.encryption.tasks;

import static java.util.Collections.unmodifiableList;

import android.annotation.Nullable;

import com.android.server.backup.encryption.chunk.ChunkHash;

import com.android.server.backup.encryption.chunking.EncryptedChunk;

import java.io.IOException;

import java.security.GeneralSecurityException;

import java.util.ArrayList;

import java.util.List;

import java.util.Set;

import javax.crypto.SecretKey;

/** Task which reads data from some source, splits it into chunks and encrypts new chunks. */

public interface BackupEncrypter {

    /** The algorithm which we use to compute the digest of the backup file plaintext. */

    String MESSAGE_DIGEST_ALGORITHM = "SHA-256";

    /**

     * Splits the backup input into encrypted chunks and encrypts new chunks.

     *

     * @param secretKey Key used to encrypt backup.

     * @param fingerprintMixerSalt Fingerprint mixer salt used for content-defined chunking during a

     *     full backup. Should be {@code null} for a key-value backup.

     * @param existingChunks Set of the SHA-256 Macs of chunks the server already has.

     * @return a result containing an array of new encrypted chunks to upload, and an ordered

     *     listing of the chunks in the backup file.

     * @throws IOException if a problem occurs reading from the backup data.

     * @throws GeneralSecurityException if there is a problem encrypting the data.

     */

    Result backup(

            SecretKey secretKey,

            @Nullable byte[] fingerprintMixerSalt,

            Set<ChunkHash> existingChunks)

            throws IOException, GeneralSecurityException;

    /**

     * The result of an incremental backup. Contains new encrypted chunks to upload, and an ordered

     * list of the chunks in the backup file.

     */

    class Result {

        private final List<ChunkHash> mAllChunks;

        private final List<EncryptedChunk> mNewChunks;

        private final byte[] mDigest;

        public Result(List<ChunkHash> allChunks, List<EncryptedChunk> newChunks, byte[] digest) {

            mAllChunks = unmodifiableList(new ArrayList<>(allChunks));

            mDigest = digest;

            mNewChunks = unmodifiableList(new ArrayList<>(newChunks));

        }

        /**

         * Returns an unmodifiable list of the hashes of all the chunks in the backup, in the order

         * they appear in the plaintext.

         */

        public List<ChunkHash> getAllChunks() {

            return mAllChunks;

        }

        /** Returns an unmodifiable list of the new chunks in the backup. */

        public List<EncryptedChunk> getNewChunks() {

            return mNewChunks;

        }

        /** Returns the message digest of the backup. */

        public byte[] getDigest() {

            return mDigest;

        }

    }

}

/*

 * Copyright (C) 2019 The Android Open Source Project

 *

 * Licensed under the Apache License, Version 2.0 (the "License");

 * you may not use this file except in compliance with the License.

 * You may obtain a copy of the License at

 *

 *      http://www.apache.org/licenses/LICENSE-2.0

 *

 * Unless required by applicable law or agreed to in writing, software

 * distributed under the License is distributed on an "AS IS" BASIS,

 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

 * See the License for the specific language governing permissions and

 * limitations under the License.

 */

package com.android.server.backup.encryption.tasks;

import android.util.Slog;

import com.android.server.backup.encryption.chunk.ChunkHash;

import com.android.server.backup.encryption.chunking.ChunkEncryptor;

import com.android.server.backup.encryption.chunking.ChunkHasher;

import com.android.server.backup.encryption.chunking.EncryptedChunk;

import com.android.server.backup.encryption.chunking.cdc.ContentDefinedChunker;

import com.android.server.backup.encryption.chunking.cdc.FingerprintMixer;

import com.android.server.backup.encryption.chunking.cdc.IsChunkBreakpoint;

import com.android.server.backup.encryption.chunking.cdc.RabinFingerprint64;

import java.io.IOException;

import java.io.InputStream;

import java.security.GeneralSecurityException;

import java.security.MessageDigest;

import java.security.SecureRandom;

import java.util.ArrayList;

import java.util.Collections;

import java.util.HashSet;

import java.util.List;

import java.util.Set;

import javax.crypto.SecretKey;

/**

 * Splits backup data into variable-sized chunks using content-defined chunking, then encrypts the

 * chunks. Given a hash of the SHA-256s of existing chunks, performs an incremental backup (i.e.,

 * only encrypts new chunks).

 */

public class BackupStreamEncrypter implements BackupEncrypter {

    private static final String TAG = "BackupStreamEncryptor";

    private final InputStream mData;

    private final int mMinChunkSizeBytes;

    private final int mMaxChunkSizeBytes;

    private final int mAverageChunkSizeBytes;

    /**

     * A new instance over the given distribution of chunk sizes.

     *

     * @param data The data to be backed up.

     * @param minChunkSizeBytes The minimum chunk size. No chunk will be smaller than this.

     * @param maxChunkSizeBytes The maximum chunk size. No chunk will be larger than this.

     * @param averageChunkSizeBytes The average chunk size. The mean size of chunks will be roughly

     *     this (with a few tens of bytes of overhead for the initialization vector and message

     *     authentication code).

     */

    public BackupStreamEncrypter(

            InputStream data,

            int minChunkSizeBytes,

            int maxChunkSizeBytes,

            int averageChunkSizeBytes) {

        this.mData = data;

        this.mMinChunkSizeBytes = minChunkSizeBytes;

        this.mMaxChunkSizeBytes = maxChunkSizeBytes;

        this.mAverageChunkSizeBytes = averageChunkSizeBytes;

    }

    @Override

    public Result backup(

            SecretKey secretKey, byte[] fingerprintMixerSalt, Set<ChunkHash> existingChunks)

            throws IOException, GeneralSecurityException {

        MessageDigest messageDigest =

                MessageDigest.getInstance(BackupEncrypter.MESSAGE_DIGEST_ALGORITHM);

        RabinFingerprint64 rabinFingerprint64 = new RabinFingerprint64();

        FingerprintMixer fingerprintMixer = new FingerprintMixer(secretKey, fingerprintMixerSalt);

        IsChunkBreakpoint isChunkBreakpoint =

                new IsChunkBreakpoint(mAverageChunkSizeBytes - mMinChunkSizeBytes);

        ContentDefinedChunker chunker =

                new ContentDefinedChunker(

                        mMinChunkSizeBytes,

                        mMaxChunkSizeBytes,

                        rabinFingerprint64,

                        fingerprintMixer,

                        isChunkBreakpoint);

        ChunkHasher chunkHasher = new ChunkHasher(secretKey);

        ChunkEncryptor encryptor = new ChunkEncryptor(secretKey, new SecureRandom());

        Set<ChunkHash> includedChunks = new HashSet<>();

        // New chunks will be added only once to this list, even if they occur multiple times.

        List<EncryptedChunk> newChunks = new ArrayList<>();

        // All chunks (including multiple occurrences) will be added to the chunkListing.

        List<ChunkHash> chunkListing = new ArrayList<>();

        includedChunks.addAll(existingChunks);

        chunker.chunkify(

                mData,

                chunk -> {

                    messageDigest.update(chunk);

                    ChunkHash key = chunkHasher.computeHash(chunk);

                    if (!includedChunks.contains(key)) {

                        newChunks.add(encryptor.encrypt(key, chunk));

                        includedChunks.add(key);

                    }

                    chunkListing.add(key);

                });

        Slog.i(

                TAG,

                String.format(

                        "Chunks: %d total, %d unique, %d new",

                        chunkListing.size(), new HashSet<>(chunkListing).size(), newChunks.size()));

        return new Result(

                Collections.unmodifiableList(chunkListing),

                Collections.unmodifiableList(newChunks),

                messageDigest.digest());

    }

}

/*

 * Copyright (C) 2019 The Android Open Source Project

 *

 * Licensed under the Apache License, Version 2.0 (the "License");

 * you may not use this file except in compliance with the License.

 * You may obtain a copy of the License at

 *

 *      http://www.apache.org/licenses/LICENSE-2.0

 *

 * Unless required by applicable law or agreed to in writing, software

 * distributed under the License is distributed on an "AS IS" BASIS,

 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

 * See the License for the specific language governing permissions and

 * limitations under the License.

 */

package com.android.server.backup.encryption.tasks;

/** Wraps any exception related to encryption which occurs during restore. */

public class EncryptedRestoreException extends Exception {

    public EncryptedRestoreException(String message) {

        super(message);

    }

    public EncryptedRestoreException(Throwable cause) {

        super(cause);

    }

    public EncryptedRestoreException(String message, Throwable cause) {

        super(message, cause);

    }

}

/*

 * Copyright (C) 2019 The Android Open Source Project

 *

 * Licensed under the Apache License, Version 2.0 (the "License");

 * you may not use this file except in compliance with the License.

 * You may obtain a copy of the License at

 *

 *      http://www.apache.org/licenses/LICENSE-2.0

 *

 * Unless required by applicable law or agreed to in writing, software

 * distributed under the License is distributed on an "AS IS" BASIS,

 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

 * See the License for the specific language governing permissions and

 * limitations under the License.

 */

package com.android.server.backup.encryption.tasks;

import static com.google.common.truth.Truth.assertThat;

import android.platform.test.annotations.Presubmit;

import com.android.server.backup.encryption.chunk.ChunkHash;

import com.android.server.backup.encryption.chunking.EncryptedChunk;

import com.android.server.backup.testing.CryptoTestUtils;

import com.android.server.backup.testing.RandomInputStream;

import com.google.common.collect.ImmutableList;

import com.google.common.collect.ImmutableSet;

import org.junit.Before;

import org.junit.Test;

import org.junit.runner.RunWith;

import org.robolectric.RobolectricTestRunner;

import java.io.ByteArrayInputStream;

import java.util.HashMap;

import java.util.HashSet;

import java.util.List;

import java.util.Random;

import javax.crypto.SecretKey;

@RunWith(RobolectricTestRunner.class)

@Presubmit

public class BackupStreamEncrypterTest {

    private static final int SALT_LENGTH = 32;

    private static final int BITS_PER_BYTE = 8;

    private static final int BYTES_PER_KILOBYTE = 1024;

    private static final int BYTES_PER_MEGABYTE = 1024 * 1024;

    private static final int MIN_CHUNK_SIZE = 2 * BYTES_PER_KILOBYTE;

    private static final int AVERAGE_CHUNK_SIZE = 4 * BYTES_PER_KILOBYTE;

    private static final int MAX_CHUNK_SIZE = 64 * BYTES_PER_KILOBYTE;

    private static final int BACKUP_SIZE = 2 * BYTES_PER_MEGABYTE;

    private static final int SMALL_BACKUP_SIZE = BYTES_PER_KILOBYTE;

    // 16 bytes for the mac. iv is encoded in a separate field.

    private static final int BYTES_OVERHEAD_PER_CHUNK = 16;

    private static final int MESSAGE_DIGEST_SIZE_IN_BYTES = 256 / BITS_PER_BYTE;

    private static final int RANDOM_SEED = 42;

    private static final double TOLERANCE = 0.1;

    private Random mRandom;

    private SecretKey mSecretKey;

    private byte[] mSalt;

    @Before

    public void setUp() throws Exception {

        mSecretKey = CryptoTestUtils.generateAesKey();

        mSalt = new byte[SALT_LENGTH];

        // Make these tests deterministic

        mRandom = new Random(RANDOM_SEED);

        mRandom.nextBytes(mSalt);

    }

    @Test

    public void testBackup_producesChunksOfTheGivenAverageSize() throws Exception {

        BackupEncrypter.Result result = runBackup(BACKUP_SIZE);

        long totalSize = 0;

        for (EncryptedChunk chunk : result.getNewChunks()) {

            totalSize += chunk.encryptedBytes().length;

        }

        double meanSize = totalSize / result.getNewChunks().size();

        double expectedChunkSize = AVERAGE_CHUNK_SIZE + BYTES_OVERHEAD_PER_CHUNK;

        assertThat(Math.abs(meanSize - expectedChunkSize) / expectedChunkSize)

                .isLessThan(TOLERANCE);

    }

    @Test

    public void testBackup_producesNoChunksSmallerThanMinSize() throws Exception {

        BackupEncrypter.Result result = runBackup(BACKUP_SIZE);

        List<EncryptedChunk> chunks = result.getNewChunks();

        // Last chunk could be smaller, depending on the file size and how it is chunked

        for (EncryptedChunk chunk : chunks.subList(0, chunks.size() - 2)) {

            assertThat(chunk.encryptedBytes().length)

                    .isAtLeast(MIN_CHUNK_SIZE + BYTES_OVERHEAD_PER_CHUNK);

        }

    }

    @Test

    public void testBackup_producesNoChunksLargerThanMaxSize() throws Exception {

        BackupEncrypter.Result result = runBackup(BACKUP_SIZE);

        List<EncryptedChunk> chunks = result.getNewChunks();

        for (EncryptedChunk chunk : chunks) {

            assertThat(chunk.encryptedBytes().length)

                    .isAtMost(MAX_CHUNK_SIZE + BYTES_OVERHEAD_PER_CHUNK);

        }

    }

    @Test

    public void testBackup_producesAFileOfTheExpectedSize() throws Exception {

        BackupEncrypter.Result result = runBackup(BACKUP_SIZE);

        HashMap<ChunkHash, EncryptedChunk> chunksBySha256 =

                chunksIndexedByKey(result.getNewChunks());

        int expectedSize = BACKUP_SIZE + result.getAllChunks().size() * BYTES_OVERHEAD_PER_CHUNK;

        int size = 0;

        for (ChunkHash byteString : result.getAllChunks()) {

            size += chunksBySha256.get(byteString).encryptedBytes().length;

        }

        assertThat(size).isEqualTo(expectedSize);

    }

    @Test

    public void testBackup_forSameFile_producesNoNewChunks() throws Exception {

        byte[] backupData = getRandomData(BACKUP_SIZE);

        BackupEncrypter.Result result = runBackup(backupData, ImmutableList.of());

        BackupEncrypter.Result incrementalResult = runBackup(backupData, result.getAllChunks());

        assertThat(incrementalResult.getNewChunks()).isEmpty();

    }

    @Test

    public void testBackup_onlyUpdatesChangedChunks() throws Exception {

        byte[] backupData = getRandomData(BACKUP_SIZE);

        BackupEncrypter.Result result = runBackup(backupData, ImmutableList.of());

        // Let's update the 2nd and 5th chunk

        backupData[positionOfChunk(result, 1)]++;

        backupData[positionOfChunk(result, 4)]++;

        BackupEncrypter.Result incrementalResult = runBackup(backupData, result.getAllChunks());

        assertThat(incrementalResult.getNewChunks()).hasSize(2);

    }

    @Test

    public void testBackup_doesNotIncludeUpdatedChunksInNewListing() throws Exception {

        byte[] backupData = getRandomData(BACKUP_SIZE);

        BackupEncrypter.Result result = runBackup(backupData, ImmutableList.of());

        // Let's update the 2nd and 5th chunk

        backupData[positionOfChunk(result, 1)]++;

        backupData[positionOfChunk(result, 4)]++;

        BackupEncrypter.Result incrementalResult = runBackup(backupData, result.getAllChunks());

        List<EncryptedChunk> newChunks = incrementalResult.getNewChunks();

        List<ChunkHash> chunkListing = result.getAllChunks();

        assertThat(newChunks).doesNotContain(chunkListing.get(1));

        assertThat(newChunks).doesNotContain(chunkListing.get(4));

    }

    @Test

    public void testBackup_includesUnchangedChunksInNewListing() throws Exception {

        byte[] backupData = getRandomData(BACKUP_SIZE);

        BackupEncrypter.Result result = runBackup(backupData, ImmutableList.of());

        // Let's update the 2nd and 5th chunk

        backupData[positionOfChunk(result, 1)]++;

        backupData[positionOfChunk(result, 4)]++;

        BackupEncrypter.Result incrementalResult = runBackup(backupData, result.getAllChunks());

        HashSet<ChunkHash> chunksPresentInIncremental =

                new HashSet<>(incrementalResult.getAllChunks());

        chunksPresentInIncremental.removeAll(result.getAllChunks());

        assertThat(chunksPresentInIncremental).hasSize(2);

    }

    @Test

    public void testBackup_forSameData_createsSameDigest() throws Exception {

        byte[] backupData = getRandomData(SMALL_BACKUP_SIZE);

        BackupEncrypter.Result result = runBackup(backupData, ImmutableList.of());

        BackupEncrypter.Result result2 = runBackup(backupData, ImmutableList.of());

        assertThat(result.getDigest()).isEqualTo(result2.getDigest());

    }

    @Test

    public void testBackup_forDifferentData_createsDifferentDigest() throws Exception {

        byte[] backup1Data = getRandomData(SMALL_BACKUP_SIZE);

        byte[] backup2Data = getRandomData(SMALL_BACKUP_SIZE);

        BackupEncrypter.Result result = runBackup(backup1Data, ImmutableList.of());

        BackupEncrypter.Result result2 = runBackup(backup2Data, ImmutableList.of());

        assertThat(result.getDigest()).isNotEqualTo(result2.getDigest());

    }

    @Test

    public void testBackup_createsDigestOf32Bytes() throws Exception {

        assertThat(runBackup(getRandomData(SMALL_BACKUP_SIZE), ImmutableList.of()).getDigest())

                .hasLength(MESSAGE_DIGEST_SIZE_IN_BYTES);

    }

    private byte[] getRandomData(int size) throws Exception {

        RandomInputStream randomInputStream = new RandomInputStream(mRandom, size);

        byte[] backupData = new byte[size];

        randomInputStream.read(backupData);

        return backupData;

    }

    private BackupEncrypter.Result runBackup(int backupSize) throws Exception {

        RandomInputStream dataStream = new RandomInputStream(mRandom, backupSize);

        BackupStreamEncrypter task =

                new BackupStreamEncrypter(

                        dataStream, MIN_CHUNK_SIZE, MAX_CHUNK_SIZE, AVERAGE_CHUNK_SIZE);

        return task.backup(mSecretKey, mSalt, ImmutableSet.of());

    }

    private BackupEncrypter.Result runBackup(byte[] data, List<ChunkHash> existingChunks)

            throws Exception {

        ByteArrayInputStream dataStream = new ByteArrayInputStream(data);

        BackupStreamEncrypter task =

                new BackupStreamEncrypter(

                        dataStream, MIN_CHUNK_SIZE, MAX_CHUNK_SIZE, AVERAGE_CHUNK_SIZE);

        return task.backup(mSecretKey, mSalt, ImmutableSet.copyOf(existingChunks));

    }

    /** Returns a {@link HashMap} of the chunks, indexed by the SHA-256 Mac key. */

    private static HashMap<ChunkHash, EncryptedChunk> chunksIndexedByKey(

            List<EncryptedChunk> chunks) {

        HashMap<ChunkHash, EncryptedChunk> chunksByKey = new HashMap<>();

        for (EncryptedChunk chunk : chunks) {

            chunksByKey.put(chunk.key(), chunk);

        }

        return chunksByKey;

    }

    /**

     * Returns the start position of the chunk in the plaintext backup data.

     *

     * @param result The result from a backup.

     * @param index The index of the chunk in question.

     * @return the start position.

     */

    private static int positionOfChunk(BackupEncrypter.Result result, int index) {

        HashMap<ChunkHash, EncryptedChunk> byKey = chunksIndexedByKey(result.getNewChunks());

        List<ChunkHash> listing = result.getAllChunks();

        int position = 0;

        for (int i = 0; i < index - 1; i++) {

            EncryptedChunk chunk = byKey.get(listing.get(i));

            position += chunk.encryptedBytes().length - BYTES_OVERHEAD_PER_CHUNK;

        }

        return position;

    }

}

/*

 * Copyright (C) 2019 The Android Open Source Project

 *

 * Licensed under the Apache License, Version 2.0 (the "License");

 * you may not use this file except in compliance with the License.

 * You may obtain a copy of the License at

 *

 *      http://www.apache.org/licenses/LICENSE-2.0

 *

 * Unless required by applicable law or agreed to in writing, software

 * distributed under the License is distributed on an "AS IS" BASIS,

 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

 * See the License for the specific language governing permissions and

 * limitations under the License.

 */

package com.android.server.backup.testing;

import static com.android.internal.util.Preconditions.checkArgument;

import java.io.IOException;

import java.io.InputStream;

import java.util.Random;

/** {@link InputStream} that generates random bytes up to a given length. For testing purposes. */

public class RandomInputStream extends InputStream {

    private static final int BYTE_MAX_VALUE = 255;

    private final Random mRandom;

    private final int mSizeBytes;

    private int mBytesRead;

    /**

     * A new instance, generating {@code sizeBytes} from {@code random} as a source.

     *

     * @param random Source of random bytes.

     * @param sizeBytes The number of bytes to generate before closing the stream.

     */

    public RandomInputStream(Random random, int sizeBytes) {

        mRandom = random;

        mSizeBytes = sizeBytes;

        mBytesRead = 0;

    }

    @Override

    public int read() throws IOException {

        if (isFinished()) {

            return -1;

        }

        mBytesRead++;

        return mRandom.nextInt(BYTE_MAX_VALUE);

    }

    @Override

    public int read(byte[] b, int off, int len) throws IOException {

        checkArgument(off + len <= b.length);

        if (isFinished()) {

            return -1;

        }

        int length = Math.min(len, mSizeBytes - mBytesRead);

        int end = off + length;

        for (int i = off; i < end; ) {

            for (int rnd = mRandom.nextInt(), n = Math.min(end - i, Integer.SIZE / Byte.SIZE);

                    n-- > 0;

                    rnd >>= Byte.SIZE) {

                b[i++] = (byte) rnd;

            }

        }

        mBytesRead += length;

        return length;

    }

    private boolean isFinished() {

        return mBytesRead >= mSizeBytes;

    }

}