Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 02e31fc5 authored by Treehugger Robot's avatar Treehugger Robot Committed by Gerrit Code Review
Browse files

Merge changes from topic "merge-incremental-adb" 

* changes:
  [adb] file sync performance on Windows
  Adding block kind to protocol for future streaming of a tree.
  [adb] Print fewer progress messages for push/pull
  [adb] Optimize fdevent machinery
  [adb] Fix incremental installation on Windows
  [adb incremental] send priority blocks first
  [adb data server] wait for installation results before terminates
  Implement the new v4 signing scheme in adb
parents 19c5cc22 47a58ecd

adb/Android.bp

+1 −0
Original line number Diff line number Diff line
@@ -298,6 +298,7 @@ cc_binary_host { 
        "client/fastdeploycallbacks.cpp",

        "client/incremental.cpp",

        "client/incremental_server.cpp",

        "client/incremental_utils.cpp",

        "shell_service_protocol.cpp",

    ],

adb/client/commandline.cpp

+40 −9
Original line number Diff line number Diff line
@@ -1443,6 +1443,26 @@ static bool _is_valid_ack_reply_fd(const int ack_reply_fd) { 
#endif

}



static bool _is_valid_os_fd(int fd) {

    // Disallow invalid FDs and stdin/out/err as well.

    if (fd < 3) {

        return false;

    }

#ifdef _WIN32

    auto handle = (HANDLE)fd;

    DWORD info = 0;

    if (GetHandleInformation(handle, &info) == 0) {

        return false;

    }

#else

    int flags = fcntl(fd, F_GETFD);

    if (flags == -1) {

        return false;

    }

#endif

    return true;

}



int adb_commandline(int argc, const char** argv) {

    bool no_daemon = false;

    bool is_daemon = false;
@@ -2009,17 +2029,28 @@ int adb_commandline(int argc, const char** argv) { 
            }

        }

    } else if (!strcmp(argv[0], "inc-server")) {

        if (argc < 3) {

            error_exit("usage: adb inc-server FD FILE1 FILE2 ...");

        if (argc < 4) {

#ifdef _WIN32

            error_exit("usage: adb inc-server CONNECTION_HANDLE OUTPUT_HANDLE FILE1 FILE2 ...");

#else

            error_exit("usage: adb inc-server CONNECTION_FD OUTPUT_FD FILE1 FILE2 ...");

#endif

        }

        int fd = atoi(argv[1]);

        if (fd < 3) {

            // Disallow invalid FDs and stdin/out/err as well.

            error_exit("Invalid fd number given: %d", fd);

        int connection_fd = atoi(argv[1]);

        if (!_is_valid_os_fd(connection_fd)) {

            error_exit("Invalid connection_fd number given: %d", connection_fd);

        }



        connection_fd = adb_register_socket(connection_fd);

        close_on_exec(connection_fd);



        int output_fd = atoi(argv[2]);

        if (!_is_valid_os_fd(output_fd)) {

            error_exit("Invalid output_fd number given: %d", output_fd);

        }

        fd = adb_register_socket(fd);

        close_on_exec(fd);

        return incremental::serve(fd, argc - 2, argv + 2);

        output_fd = adb_register_socket(output_fd);

        close_on_exec(output_fd);

        return incremental::serve(connection_fd, output_fd, argc - 3, argv + 3);

    }



    error_exit("unknown command %s", argv[0]);

adb/client/file_sync_client.cpp

+20 −1
Original line number Diff line number Diff line
@@ -53,6 +53,8 @@ 
#include <android-base/strings.h>

#include <android-base/stringprintf.h>



using namespace std::literals;



typedef void(sync_ls_cb)(unsigned mode, uint64_t size, uint64_t time, const char* name);



struct syncsendbuf {
@@ -113,6 +115,11 @@ struct TransferLedger { 
    uint64_t bytes_expected;

    bool expect_multiple_files;



  private:

    std::string last_progress_str;

    std::chrono::steady_clock::time_point last_progress_time;



  public:

    TransferLedger() {

        Reset();

    }
@@ -132,6 +139,8 @@ struct TransferLedger { 
        files_skipped = 0;

        bytes_transferred = 0;

        bytes_expected = 0;

        last_progress_str.clear();

        last_progress_time = {};

    }



    std::string TransferRate() {
@@ -151,6 +160,12 @@ struct TransferLedger { 


    void ReportProgress(LinePrinter& lp, const std::string& file, uint64_t file_copied_bytes,

                        uint64_t file_total_bytes) {

        static constexpr auto kProgressReportInterval = 100ms;



        auto now = std::chrono::steady_clock::now();

        if (now < last_progress_time + kProgressReportInterval) {

            return;

        }

        char overall_percentage_str[5] = "?";

        if (bytes_expected != 0 && bytes_transferred <= bytes_expected) {

            int overall_percentage = static_cast<int>(bytes_transferred * 100 / bytes_expected);
@@ -181,7 +196,11 @@ struct TransferLedger { 
                    android::base::StringPrintf("[%4s] %s", overall_percentage_str, file.c_str());

            }

        }

        if (output != last_progress_str) {

            lp.Print(output, LinePrinter::LineType::INFO);

            last_progress_str = std::move(output);

            last_progress_time = now;

        }

    }



    void ReportTransferRate(LinePrinter& lp, const std::string& name, TransferDirection direction) {

adb/client/incremental.cpp

+72 −22
Original line number Diff line number Diff line
@@ -41,37 +41,35 @@ using Size = int64_t; 


static inline int32_t read_int32(borrowed_fd fd) {

    int32_t result;

    ReadFully(fd, &result, sizeof(result));

    return result;

}



static inline int32_t read_be_int32(borrowed_fd fd) {

    return int32_t(be32toh(read_int32(fd)));

    return ReadFdExactly(fd, &result, sizeof(result)) ? result : -1;

}



static inline void append_int(borrowed_fd fd, std::vector<char>* bytes) {

    int32_t be_val = read_int32(fd);

    int32_t le_val = read_int32(fd);

    auto old_size = bytes->size();

    bytes->resize(old_size + sizeof(be_val));

    memcpy(bytes->data() + old_size, &be_val, sizeof(be_val));

    bytes->resize(old_size + sizeof(le_val));

    memcpy(bytes->data() + old_size, &le_val, sizeof(le_val));

}



static inline void append_bytes_with_size(borrowed_fd fd, std::vector<char>* bytes) {

    int32_t be_size = read_int32(fd);

    int32_t size = int32_t(be32toh(be_size));

    int32_t le_size = read_int32(fd);

    if (le_size < 0) {

        return;

    }

    int32_t size = int32_t(le32toh(le_size));

    auto old_size = bytes->size();

    bytes->resize(old_size + sizeof(be_size) + size);

    memcpy(bytes->data() + old_size, &be_size, sizeof(be_size));

    ReadFully(fd, bytes->data() + old_size + sizeof(be_size), size);

    bytes->resize(old_size + sizeof(le_size) + size);

    memcpy(bytes->data() + old_size, &le_size, sizeof(le_size));

    ReadFdExactly(fd, bytes->data() + old_size + sizeof(le_size), size);

}



static inline std::pair<std::vector<char>, int32_t> read_id_sig_headers(borrowed_fd fd) {

    std::vector<char> result;

    append_int(fd, &result);              // version

    append_bytes_with_size(fd, &result);  // verityRootHash

    append_bytes_with_size(fd, &result);  // v3Digest

    append_bytes_with_size(fd, &result);  // pkcs7SignatureBlock

    auto tree_size = read_be_int32(fd);   // size of the verity tree

    append_bytes_with_size(fd, &result);  // hashingInfo

    append_bytes_with_size(fd, &result);  // signingInfo

    auto le_tree_size = read_int32(fd);

    auto tree_size = int32_t(le32toh(le_tree_size));  // size of the verity tree

    return {std::move(result), tree_size};

}
@@ -197,20 +195,72 @@ std::optional<Process> install(std::vector<std::string> files) { 
    auto fd_param = std::to_string(osh);

#endif



    // pipe for child process to write output

    int print_fds[2];

    if (adb_socketpair(print_fds) != 0) {

        fprintf(stderr, "Failed to create socket pair for child to print to parent\n");

        return {};

    }

    auto [pipe_read_fd, pipe_write_fd] = print_fds;

    auto pipe_write_fd_param = std::to_string(intptr_t(adb_get_os_handle(pipe_write_fd)));

    close_on_exec(pipe_read_fd);



    std::vector<std::string> args(std::move(files));

    args.insert(args.begin(), {"inc-server", fd_param});

    auto child = adb_launch_process(adb_path, std::move(args), {connection_fd.get()});

    args.insert(args.begin(), {"inc-server", fd_param, pipe_write_fd_param});

    auto child =

            adb_launch_process(adb_path, std::move(args), {connection_fd.get(), pipe_write_fd});

    if (!child) {

        fprintf(stderr, "adb: failed to fork: %s\n", strerror(errno));

        return {};

    }



    adb_close(pipe_write_fd);



    auto killOnExit = [](Process* p) { p->kill(); };

    std::unique_ptr<Process, decltype(killOnExit)> serverKiller(&child, killOnExit);

    // TODO: Terminate server process if installation fails.

    serverKiller.release();



    Result result = wait_for_installation(pipe_read_fd);

    adb_close(pipe_read_fd);



    if (result == Result::Success) {

        // adb client exits now but inc-server can continue

        serverKiller.release();

    }

    return child;

}



Result wait_for_installation(int read_fd) {

    static constexpr int maxMessageSize = 256;

    std::vector<char> child_stdout(CHUNK_SIZE);

    int bytes_read;

    int buf_size = 0;

    // TODO(b/150865433): optimize child's output parsing

    while ((bytes_read = adb_read(read_fd, child_stdout.data() + buf_size,

                                  child_stdout.size() - buf_size)) > 0) {

        // print to parent's stdout

        fprintf(stdout, "%.*s", bytes_read, child_stdout.data() + buf_size);



        buf_size += bytes_read;

        const std::string_view stdout_str(child_stdout.data(), buf_size);

        // wait till installation either succeeds or fails

        if (stdout_str.find("Success") != std::string::npos) {

            return Result::Success;

        }

        // on failure, wait for full message

        static constexpr auto failure_msg_head = "Failure ["sv;

        if (const auto begin_itr = stdout_str.find(failure_msg_head);

            begin_itr != std::string::npos) {

            if (buf_size >= maxMessageSize) {

                return Result::Failure;

            }

            const auto end_itr = stdout_str.rfind("]");

            if (end_itr != std::string::npos && end_itr >= begin_itr + failure_msg_head.size()) {

                return Result::Failure;

            }

        }

        child_stdout.resize(buf_size + CHUNK_SIZE);

    }

    return Result::None;

}



}  // namespace incremental

adb/client/incremental.h

+3 −0
Original line number Diff line number Diff line
@@ -27,4 +27,7 @@ namespace incremental { 


std::optional<Process> install(std::vector<std::string> files);



enum class Result { Success, Failure, None };

Result wait_for_installation(int read_fd);



}  // namespace incremental