Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 939fc19a authored by Josh Gao's avatar Josh Gao
Browse files

adb: implement compression for file sync. 

This improves performance when syncing by up to 2x (remote cuttlefish
goes from 11.9 MB/s to 21.3 MB/s, blueline over USB 2.0 from 36 MB/s
to 70 MB/s).

This results in a slight drop in push speeds over USB 3.0 (125 -> 115
MB/s on blueline), presumably because we're compressing and extracting
on only a single thread, but the gains over lower bandwidth transports
make this worth it to submit this now and parallelize later.

Bug: https://issuetracker.google.com/150827486
Test: ADB_COMPRESSION={0, 1} test_device.py (with new/old adbd)
Change-Id: Ic2a0c974f1b6efecda115f87d336e3caac810035
parent 5e4b94d4

adb/Android.bp

+4 −0
Original line number Diff line number Diff line
@@ -317,6 +317,7 @@ cc_binary_host { 
        "libandroidfw",

        "libapp_processes_protos_full",

        "libbase",

        "libbrotli",

        "libcutils",

        "libcrypto_utils",

        "libcrypto",
@@ -469,6 +470,7 @@ cc_library { 
    static_libs: [

        "libadbconnection_server",

        "libadbd_core",

        "libbrotli",

        "libdiagnose_usb",

    ],
@@ -567,6 +569,7 @@ cc_library { 
    },



    static_libs: [

        "libbrotli",

        "libcutils_sockets",

        "libdiagnose_usb",

        "libmdnssd",
@@ -606,6 +609,7 @@ cc_binary { 
        "libapp_processes_protos_lite",

        "libasyncio",

        "libbase",

        "libbrotli",

        "libcap",

        "libcrypto_utils",

        "libcutils_sockets",

adb/brotli_utils.h

0 → 100644
+144 −0
Original line number Diff line number Diff line 
/*

 * Copyright (C) 2020 The Android Open Source Project

 *

 * Licensed under the Apache License, Version 2.0 (the "License");

 * you may not use this file except in compliance with the License.

 * You may obtain a copy of the License at

 *

 *      http://www.apache.org/licenses/LICENSE-2.0

 *

 * Unless required by applicable law or agreed to in writing, software

 * distributed under the License is distributed on an "AS IS" BASIS,

 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

 * See the License for the specific language governing permissions and

 * limitations under the License.

 */



#pragma once



#include <span>



#include <brotli/decode.h>

#include <brotli/encode.h>



#include "types.h"



enum class BrotliDecodeResult {

    Error,

    Done,

    NeedInput,

    MoreOutput,

};



struct BrotliDecoder {

    explicit BrotliDecoder(std::span<char> output_buffer)

        : output_buffer_(output_buffer),

          decoder_(BrotliDecoderCreateInstance(nullptr, nullptr, nullptr),

                   BrotliDecoderDestroyInstance) {}



    void Append(Block&& block) { input_buffer_.append(std::move(block)); }



    BrotliDecodeResult Decode(std::span<char>* output) {

        size_t available_in = input_buffer_.front_size();

        const uint8_t* next_in = reinterpret_cast<const uint8_t*>(input_buffer_.front_data());



        size_t available_out = output_buffer_.size();

        uint8_t* next_out = reinterpret_cast<uint8_t*>(output_buffer_.data());



        BrotliDecoderResult r = BrotliDecoderDecompressStream(

                decoder_.get(), &available_in, &next_in, &available_out, &next_out, nullptr);



        size_t bytes_consumed = input_buffer_.front_size() - available_in;

        input_buffer_.drop_front(bytes_consumed);



        size_t bytes_emitted = output_buffer_.size() - available_out;

        *output = std::span<char>(output_buffer_.data(), bytes_emitted);



        switch (r) {

            case BROTLI_DECODER_RESULT_SUCCESS:

                return BrotliDecodeResult::Done;

            case BROTLI_DECODER_RESULT_ERROR:

                return BrotliDecodeResult::Error;

            case BROTLI_DECODER_RESULT_NEEDS_MORE_INPUT:

                // Brotli guarantees as one of its invariants that if it returns NEEDS_MORE_INPUT,

                // it will consume the entire input buffer passed in, so we don't have to worry

                // about bytes left over in the front block with more input remaining.

                return BrotliDecodeResult::NeedInput;

            case BROTLI_DECODER_RESULT_NEEDS_MORE_OUTPUT:

                return BrotliDecodeResult::MoreOutput;

        }

    }



  private:

    IOVector input_buffer_;

    std::span<char> output_buffer_;

    std::unique_ptr<BrotliDecoderState, void (*)(BrotliDecoderState*)> decoder_;

};



enum class BrotliEncodeResult {

    Error,

    Done,

    NeedInput,

    MoreOutput,

};



template <size_t OutputBlockSize>

struct BrotliEncoder {

    explicit BrotliEncoder()

        : output_block_(OutputBlockSize),

          output_bytes_left_(OutputBlockSize),

          encoder_(BrotliEncoderCreateInstance(nullptr, nullptr, nullptr),

                   BrotliEncoderDestroyInstance) {

        BrotliEncoderSetParameter(encoder_.get(), BROTLI_PARAM_QUALITY, 1);

    }



    void Append(Block input) { input_buffer_.append(std::move(input)); }

    void Finish() { finished_ = true; }



    BrotliEncodeResult Encode(Block* output) {

        output->clear();

        while (true) {

            size_t available_in = input_buffer_.front_size();

            const uint8_t* next_in = reinterpret_cast<const uint8_t*>(input_buffer_.front_data());



            size_t available_out = output_bytes_left_;

            uint8_t* next_out = reinterpret_cast<uint8_t*>(output_block_.data() +

                                                           (OutputBlockSize - output_bytes_left_));



            BrotliEncoderOperation op = BROTLI_OPERATION_PROCESS;

            if (finished_) {

                op = BROTLI_OPERATION_FINISH;

            }



            if (!BrotliEncoderCompressStream(encoder_.get(), op, &available_in, &next_in,

                                             &available_out, &next_out, nullptr)) {

                return BrotliEncodeResult::Error;

            }



            size_t bytes_consumed = input_buffer_.front_size() - available_in;

            input_buffer_.drop_front(bytes_consumed);



            output_bytes_left_ = available_out;



            if (BrotliEncoderIsFinished(encoder_.get())) {

                output_block_.resize(OutputBlockSize - output_bytes_left_);

                *output = std::move(output_block_);

                return BrotliEncodeResult::Done;

            } else if (output_bytes_left_ == 0) {

                *output = std::move(output_block_);

                output_block_.resize(OutputBlockSize);

                output_bytes_left_ = OutputBlockSize;

                return BrotliEncodeResult::MoreOutput;

            } else if (input_buffer_.empty()) {

                return BrotliEncodeResult::NeedInput;

            }

        }

    }



  private:

    bool finished_ = false;

    IOVector input_buffer_;

    Block output_block_;

    size_t output_bytes_left_;

    std::unique_ptr<BrotliEncoderState, void (*)(BrotliEncoderState*)> encoder_;

};

adb/client/adb_install.cpp

+1 −1
Original line number Diff line number Diff line
@@ -286,7 +286,7 @@ static int install_app_legacy(int argc, const char** argv, bool use_fastdeploy) 
        }

    }



    if (do_sync_push(apk_file, apk_dest.c_str(), false)) {

    if (do_sync_push(apk_file, apk_dest.c_str(), false, true)) {

        result = pm_command(argc, argv);

        delete_device_file(apk_dest);

    }

adb/client/commandline.cpp

+58 −18
Original line number Diff line number Diff line
@@ -129,15 +129,21 @@ static void help() { 
        " reverse --remove-all     remove all reverse socket connections from device\n"

        "\n"

        "file transfer:\n"

        " push [--sync] LOCAL... REMOTE\n"

        " push [--sync] [-zZ] LOCAL... REMOTE\n"

        "     copy local files/directories to device\n"

        "     --sync: only push files that are newer on the host than the device\n"

        " pull [-a] REMOTE... LOCAL\n"

        "     -z: enable compression\n"

        "     -Z: disable compression\n"

        " pull [-azZ] REMOTE... LOCAL\n"

        "     copy files/dirs from device\n"

        "     -a: preserve file timestamp and mode\n"

        " sync [all|data|odm|oem|product|system|system_ext|vendor]\n"

        "     -z: enable compression\n"

        "     -Z: disable compression\n"

        " sync [-lzZ] [all|data|odm|oem|product|system|system_ext|vendor]\n"

        "     sync a local build from $ANDROID_PRODUCT_OUT to the device (default all)\n"

        "     -l: list files that would be copied, but don't copy them\n"

        "     -z: enable compression\n"

        "     -Z: disable compression\n"

        "\n"

        "shell:\n"

        " shell [-e ESCAPE] [-n] [-Tt] [-x] [COMMAND...]\n"
@@ -1309,8 +1315,12 @@ static int restore(int argc, const char** argv) { 
}



static void parse_push_pull_args(const char** arg, int narg, std::vector<const char*>* srcs,

                                 const char** dst, bool* copy_attrs, bool* sync) {

                                 const char** dst, bool* copy_attrs, bool* sync, bool* compressed) {

    *copy_attrs = false;

    const char* adb_compression = getenv("ADB_COMPRESSION");

    if (adb_compression && strcmp(adb_compression, "0") == 0) {

        *compressed = false;

    }



    srcs->clear();

    bool ignore_flags = false;
@@ -1322,6 +1332,14 @@ static void parse_push_pull_args(const char** arg, int narg, std::vector<const c 
                // Silently ignore for backwards compatibility.

            } else if (!strcmp(*arg, "-a")) {

                *copy_attrs = true;

            } else if (!strcmp(*arg, "-z")) {

                if (compressed != nullptr) {

                    *compressed = true;

                }

            } else if (!strcmp(*arg, "-Z")) {

                if (compressed != nullptr) {

                    *compressed = false;

                }

            } else if (!strcmp(*arg, "--sync")) {

                if (sync != nullptr) {

                    *sync = true;
@@ -1876,20 +1894,22 @@ int adb_commandline(int argc, const char** argv) { 
    } else if (!strcmp(argv[0], "push")) {

        bool copy_attrs = false;

        bool sync = false;

        bool compressed = true;

        std::vector<const char*> srcs;

        const char* dst = nullptr;



        parse_push_pull_args(&argv[1], argc - 1, &srcs, &dst, &copy_attrs, &sync);

        parse_push_pull_args(&argv[1], argc - 1, &srcs, &dst, &copy_attrs, &sync, &compressed);

        if (srcs.empty() || !dst) error_exit("push requires an argument");

        return do_sync_push(srcs, dst, sync) ? 0 : 1;

        return do_sync_push(srcs, dst, sync, compressed) ? 0 : 1;

    } else if (!strcmp(argv[0], "pull")) {

        bool copy_attrs = false;

        bool compressed = true;

        std::vector<const char*> srcs;

        const char* dst = ".";



        parse_push_pull_args(&argv[1], argc - 1, &srcs, &dst, &copy_attrs, nullptr);

        parse_push_pull_args(&argv[1], argc - 1, &srcs, &dst, &copy_attrs, nullptr, &compressed);

        if (srcs.empty()) error_exit("pull requires an argument");

        return do_sync_pull(srcs, dst, copy_attrs) ? 0 : 1;

        return do_sync_pull(srcs, dst, copy_attrs, compressed) ? 0 : 1;

    } else if (!strcmp(argv[0], "install")) {

        if (argc < 2) error_exit("install requires an argument");

        return install_app(argc, argv);
@@ -1905,18 +1925,38 @@ int adb_commandline(int argc, const char** argv) { 
    } else if (!strcmp(argv[0], "sync")) {

        std::string src;

        bool list_only = false;

        if (argc < 2) {

            // No partition specified: sync all of them.

        } else if (argc >= 2 && strcmp(argv[1], "-l") == 0) {

        bool compressed = true;



        const char* adb_compression = getenv("ADB_COMPRESSION");

        if (adb_compression && strcmp(adb_compression, "0") == 0) {

            compressed = false;

        }



        int opt;

        while ((opt = getopt(argc, const_cast<char**>(argv), "lzZ")) != -1) {

            switch (opt) {

                case 'l':

                    list_only = true;

            if (argc == 3) src = argv[2];

        } else if (argc == 2) {

            src = argv[1];

                    break;

                case 'z':

                    compressed = true;

                    break;

                case 'Z':

                    compressed = false;

                    break;

                default:

                    error_exit("usage: adb sync [-lzZ] [PARTITION]");

            }

        }



        if (optind == argc) {

            src = "all";

        } else if (optind + 1 == argc) {

            src = argv[optind];

        } else {

            error_exit("usage: adb sync [-l] [PARTITION]");

            error_exit("usage: adb sync [-lzZ] [PARTITION]");

        }



        if (src.empty()) src = "all";

        std::vector<std::string> partitions{"data",   "odm",        "oem",   "product",

                                            "system", "system_ext", "vendor"};

        bool found = false;
@@ -1925,7 +1965,7 @@ int adb_commandline(int argc, const char** argv) { 
                std::string src_dir{product_file(partition)};

                if (!directory_exists(src_dir)) continue;

                found = true;

                if (!do_sync_sync(src_dir, "/" + partition, list_only)) return 1;

                if (!do_sync_sync(src_dir, "/" + partition, list_only, compressed)) return 1;

            }

        }

        if (!found) error_exit("don't know how to sync %s partition", src.c_str());

adb/client/fastdeploy.cpp

+1 −1
Original line number Diff line number Diff line
@@ -112,7 +112,7 @@ static void push_to_device(const void* data, size_t byte_count, const char* dst, 
    // but can't be removed until after the push.

    unix_close(tf.release());



    if (!do_sync_push(srcs, dst, sync)) {

    if (!do_sync_push(srcs, dst, sync, true)) {

        error_exit("Failed to push fastdeploy agent to device.");

    }

}