Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 5949fccc authored by Josh Gao's avatar Josh Gao
Browse files

adb: add dry-run option to push/sync. 

Make it easier to benchmark file sync performance by ignoring the file
system.

Bug: https://issuetracker.google.com/150827486
Test: test_device.py
Change-Id: Icfa4b28eb5206f1914c0c163833d070a3748c3ea
parent ec44d35f

adb/client/adb_install.cpp

+1 −1
Original line number Diff line number Diff line
@@ -290,7 +290,7 @@ static int install_app_legacy(int argc, const char** argv, bool use_fastdeploy) 
        }

    }



    if (do_sync_push(apk_file, apk_dest.c_str(), false, CompressionType::Any)) {

    if (do_sync_push(apk_file, apk_dest.c_str(), false, CompressionType::Any, false)) {

        result = pm_command(argc, argv);

        delete_device_file(apk_dest);

    }

adb/client/commandline.cpp

+21 −8
Original line number Diff line number Diff line
@@ -132,6 +132,7 @@ static void help() { 
        " push [--sync] [-z ALGORITHM] [-Z] LOCAL... REMOTE\n"

        "     copy local files/directories to device\n"

        "     --sync: only push files that are newer on the host than the device\n"

        "     -n: dry run: push files to device without storing to the filesystem\n"

        "     -z: enable compression with a specified algorithm (any, none, brotli)\n"

        "     -Z: disable compression\n"

        " pull [-a] [-z ALGORITHM] [-Z] REMOTE... LOCAL\n"
@@ -141,6 +142,7 @@ static void help() { 
        "     -Z: disable compression\n"

        " sync [-l] [-z ALGORITHM] [-Z] [all|data|odm|oem|product|system|system_ext|vendor]\n"

        "     sync a local build from $ANDROID_PRODUCT_OUT to the device (default all)\n"

        "     -n: dry run: push files to device without storing to the filesystem\n"

        "     -l: list files that would be copied, but don't copy them\n"

        "     -z: enable compression with a specified algorithm (any, none, brotli)\n"

        "     -Z: disable compression\n"
@@ -1340,7 +1342,7 @@ static CompressionType parse_compression_type(const std::string& str, bool allow 


static void parse_push_pull_args(const char** arg, int narg, std::vector<const char*>* srcs,

                                 const char** dst, bool* copy_attrs, bool* sync,

                                 CompressionType* compression) {

                                 CompressionType* compression, bool* dry_run) {

    *copy_attrs = false;

    if (const char* adb_compression = getenv("ADB_COMPRESSION")) {

        *compression = parse_compression_type(adb_compression, true);
@@ -1364,6 +1366,8 @@ static void parse_push_pull_args(const char** arg, int narg, std::vector<const c 
                --narg;

            } else if (!strcmp(*arg, "-Z")) {

                *compression = CompressionType::None;

            } else if (dry_run && !strcmp(*arg, "-n")) {

                *dry_run = true;

            } else if (!strcmp(*arg, "--sync")) {

                if (sync != nullptr) {

                    *sync = true;
@@ -1918,20 +1922,23 @@ int adb_commandline(int argc, const char** argv) { 
    } else if (!strcmp(argv[0], "push")) {

        bool copy_attrs = false;

        bool sync = false;

        bool dry_run = false;

        CompressionType compression = CompressionType::Any;

        std::vector<const char*> srcs;

        const char* dst = nullptr;



        parse_push_pull_args(&argv[1], argc - 1, &srcs, &dst, &copy_attrs, &sync, &compression);

        parse_push_pull_args(&argv[1], argc - 1, &srcs, &dst, &copy_attrs, &sync, &compression,

                             &dry_run);

        if (srcs.empty() || !dst) error_exit("push requires an argument");

        return do_sync_push(srcs, dst, sync, compression) ? 0 : 1;

        return do_sync_push(srcs, dst, sync, compression, dry_run) ? 0 : 1;

    } else if (!strcmp(argv[0], "pull")) {

        bool copy_attrs = false;

        CompressionType compression = CompressionType::Any;

        std::vector<const char*> srcs;

        const char* dst = ".";



        parse_push_pull_args(&argv[1], argc - 1, &srcs, &dst, &copy_attrs, nullptr, &compression);

        parse_push_pull_args(&argv[1], argc - 1, &srcs, &dst, &copy_attrs, nullptr, &compression,

                             nullptr);

        if (srcs.empty()) error_exit("pull requires an argument");

        return do_sync_pull(srcs, dst, copy_attrs, compression) ? 0 : 1;

    } else if (!strcmp(argv[0], "install")) {
@@ -1949,6 +1956,7 @@ int adb_commandline(int argc, const char** argv) { 
    } else if (!strcmp(argv[0], "sync")) {

        std::string src;

        bool list_only = false;

        bool dry_run = false;

        CompressionType compression = CompressionType::Any;



        if (const char* adb_compression = getenv("ADB_COMPRESSION"); adb_compression) {
@@ -1956,11 +1964,14 @@ int adb_commandline(int argc, const char** argv) { 
        }



        int opt;

        while ((opt = getopt(argc, const_cast<char**>(argv), "lz:Z")) != -1) {

        while ((opt = getopt(argc, const_cast<char**>(argv), "lnz:Z")) != -1) {

            switch (opt) {

                case 'l':

                    list_only = true;

                    break;

                case 'n':

                    dry_run = true;

                    break;

                case 'z':

                    compression = parse_compression_type(optarg, false);

                    break;
@@ -1968,7 +1979,7 @@ int adb_commandline(int argc, const char** argv) { 
                    compression = CompressionType::None;

                    break;

                default:

                    error_exit("usage: adb sync [-l] [-z ALGORITHM] [-Z] [PARTITION]");

                    error_exit("usage: adb sync [-l] [-n]  [-z ALGORITHM] [-Z] [PARTITION]");

            }

        }
@@ -1977,7 +1988,7 @@ int adb_commandline(int argc, const char** argv) { 
        } else if (optind + 1 == argc) {

            src = argv[optind];

        } else {

            error_exit("usage: adb sync [-l] [-z ALGORITHM] [-Z] [PARTITION]");

            error_exit("usage: adb sync [-l] [-n] [-z ALGORITHM] [-Z] [PARTITION]");

        }



        std::vector<std::string> partitions{"data",   "odm",        "oem",   "product",
@@ -1988,7 +1999,9 @@ int adb_commandline(int argc, const char** argv) { 
                std::string src_dir{product_file(partition)};

                if (!directory_exists(src_dir)) continue;

                found = true;

                if (!do_sync_sync(src_dir, "/" + partition, list_only, compression)) return 1;

                if (!do_sync_sync(src_dir, "/" + partition, list_only, compression, dry_run)) {

                    return 1;

                }

            }

        }

        if (!found) error_exit("don't know how to sync %s partition", src.c_str());

adb/client/fastdeploy.cpp

+1 −1
Original line number Diff line number Diff line
@@ -112,7 +112,7 @@ static void push_to_device(const void* data, size_t byte_count, const char* dst, 
    // but can't be removed until after the push.

    unix_close(tf.release());



    if (!do_sync_push(srcs, dst, sync, CompressionType::Any)) {

    if (!do_sync_push(srcs, dst, sync, CompressionType::Any, false)) {

        error_exit("Failed to push fastdeploy agent to device.");

    }

}

adb/client/file_sync_client.cpp

+38 −15
Original line number Diff line number Diff line
@@ -238,6 +238,7 @@ class SyncConnection { 
            have_sendrecv_v2_ = CanUseFeature(features_, kFeatureSendRecv2);

            have_sendrecv_v2_brotli_ = CanUseFeature(features_, kFeatureSendRecv2Brotli);

            have_sendrecv_v2_lz4_ = CanUseFeature(features_, kFeatureSendRecv2LZ4);

            have_sendrecv_v2_dry_run_send_ = CanUseFeature(features_, kFeatureSendRecv2DryRunSend);

            fd.reset(adb_connect("sync:", &error));

            if (fd < 0) {

                Error("connect failed: %s", error.c_str());
@@ -264,6 +265,7 @@ class SyncConnection { 
    bool HaveSendRecv2() const { return have_sendrecv_v2_; }

    bool HaveSendRecv2Brotli() const { return have_sendrecv_v2_brotli_; }

    bool HaveSendRecv2LZ4() const { return have_sendrecv_v2_lz4_; }

    bool HaveSendRecv2DryRunSend() const { return have_sendrecv_v2_dry_run_send_; }



    // Resolve a compression type which might be CompressionType::Any to a specific compression

    // algorithm.
@@ -340,7 +342,7 @@ class SyncConnection { 
        return WriteFdExactly(fd, buf.data(), buf.size());

    }



    bool SendSend2(std::string_view path, mode_t mode, CompressionType compression) {

    bool SendSend2(std::string_view path, mode_t mode, CompressionType compression, bool dry_run) {

        if (path.length() > 1024) {

            Error("SendRequest failed: path too long: %zu", path.length());

            errno = ENAMETOOLONG;
@@ -373,6 +375,10 @@ class SyncConnection { 
                LOG(FATAL) << "unexpected CompressionType::Any";

        }



        if (dry_run) {

            msg.send_v2_setup.flags |= kSyncFlagDryRun;

        }



        buf.resize(sizeof(SyncRequest) + path.length() + sizeof(msg.send_v2_setup));



        void* p = buf.data();
@@ -541,7 +547,12 @@ class SyncConnection { 
    // difference to "adb sync" performance.

    bool SendSmallFile(const std::string& path, mode_t mode, const std::string& lpath,

                       const std::string& rpath, unsigned mtime, const char* data,

                       size_t data_length) {

                       size_t data_length, bool dry_run) {

        if (dry_run) {

            // We need to use send v2 for dry run.

            return SendLargeFile(path, mode, lpath, rpath, mtime, CompressionType::None, dry_run);

        }



        std::string path_and_mode = android::base::StringPrintf("%s,%d", path.c_str(), mode);

        if (path_and_mode.length() > 1024) {

            Error("SendSmallFile failed: path too long: %zu", path_and_mode.length());
@@ -581,14 +592,20 @@ class SyncConnection { 
    }



    bool SendLargeFile(const std::string& path, mode_t mode, const std::string& lpath,

                       const std::string& rpath, unsigned mtime, CompressionType compression) {

                       const std::string& rpath, unsigned mtime, CompressionType compression,

                       bool dry_run) {

        if (dry_run && !HaveSendRecv2DryRunSend()) {

            Error("dry-run not supported by the device");

            return false;

        }



        if (!HaveSendRecv2()) {

            return SendLargeFileLegacy(path, mode, lpath, rpath, mtime);

        }



        compression = ResolveCompressionType(compression);



        if (!SendSend2(path, mode, compression)) {

        if (!SendSend2(path, mode, compression, dry_run)) {

            Error("failed to send ID_SEND_V2 message '%s': %s", path.c_str(), strerror(errno));

            return false;

        }
@@ -908,6 +925,7 @@ class SyncConnection { 
    bool have_sendrecv_v2_;

    bool have_sendrecv_v2_brotli_;

    bool have_sendrecv_v2_lz4_;

    bool have_sendrecv_v2_dry_run_send_;



    TransferLedger global_ledger_;

    TransferLedger current_ledger_;
@@ -989,7 +1007,8 @@ static bool sync_stat_fallback(SyncConnection& sc, const std::string& path, stru 
}



static bool sync_send(SyncConnection& sc, const std::string& lpath, const std::string& rpath,

                      unsigned mtime, mode_t mode, bool sync, CompressionType compression) {

                      unsigned mtime, mode_t mode, bool sync, CompressionType compression,

                      bool dry_run) {

    if (sync) {

        struct stat st;

        if (sync_lstat(sc, rpath, &st)) {
@@ -1010,7 +1029,7 @@ static bool sync_send(SyncConnection& sc, const std::string& lpath, const std::s 
        }

        buf[data_length++] = '\0';



        if (!sc.SendSmallFile(rpath, mode, lpath, rpath, mtime, buf, data_length)) {

        if (!sc.SendSmallFile(rpath, mode, lpath, rpath, mtime, buf, data_length, dry_run)) {

            return false;

        }

        return sc.ReadAcknowledgements();
@@ -1028,11 +1047,12 @@ static bool sync_send(SyncConnection& sc, const std::string& lpath, const std::s 
            sc.Error("failed to read all of '%s': %s", lpath.c_str(), strerror(errno));

            return false;

        }

        if (!sc.SendSmallFile(rpath, mode, lpath, rpath, mtime, data.data(), data.size())) {

        if (!sc.SendSmallFile(rpath, mode, lpath, rpath, mtime, data.data(), data.size(),

                              dry_run)) {

            return false;

        }

    } else {

        if (!sc.SendLargeFile(rpath, mode, lpath, rpath, mtime, compression)) {

        if (!sc.SendLargeFile(rpath, mode, lpath, rpath, mtime, compression, dry_run)) {

            return false;

        }

    }
@@ -1284,7 +1304,7 @@ static bool is_root_dir(std::string_view path) { 


static bool copy_local_dir_remote(SyncConnection& sc, std::string lpath, std::string rpath,

                                  bool check_timestamps, bool list_only,

                                  CompressionType compression) {

                                  CompressionType compression, bool dry_run) {

    sc.NewTransfer();



    // Make sure that both directory paths end in a slash.
@@ -1366,7 +1386,8 @@ static bool copy_local_dir_remote(SyncConnection& sc, std::string lpath, std::st 
            if (list_only) {

                sc.Println("would push: %s -> %s", ci.lpath.c_str(), ci.rpath.c_str());

            } else {

                if (!sync_send(sc, ci.lpath, ci.rpath, ci.time, ci.mode, false, compression)) {

                if (!sync_send(sc, ci.lpath, ci.rpath, ci.time, ci.mode, false, compression,

                               dry_run)) {

                    return false;

                }

            }
@@ -1382,7 +1403,7 @@ static bool copy_local_dir_remote(SyncConnection& sc, std::string lpath, std::st 
}



bool do_sync_push(const std::vector<const char*>& srcs, const char* dst, bool sync,

                  CompressionType compression) {

                  CompressionType compression, bool dry_run) {

    SyncConnection sc;

    if (!sc.IsValid()) return false;
@@ -1447,7 +1468,8 @@ bool do_sync_push(const std::vector<const char*>& srcs, const char* dst, bool sy 
                dst_dir.append(android::base::Basename(src_path));

            }



            success &= copy_local_dir_remote(sc, src_path, dst_dir, sync, false, compression);

            success &=

                    copy_local_dir_remote(sc, src_path, dst_dir, sync, false, compression, dry_run);

            continue;

        } else if (!should_push_file(st.st_mode)) {

            sc.Warning("skipping special file '%s' (mode = 0o%o)", src_path, st.st_mode);
@@ -1468,7 +1490,8 @@ bool do_sync_push(const std::vector<const char*>& srcs, const char* dst, bool sy 


        sc.NewTransfer();

        sc.SetExpectedTotalBytes(st.st_size);

        success &= sync_send(sc, src_path, dst_path, st.st_mtime, st.st_mode, sync, compression);

        success &= sync_send(sc, src_path, dst_path, st.st_mtime, st.st_mode, sync, compression,

                             dry_run);

        sc.ReportTransferRate(src_path, TransferDirection::push);

    }
@@ -1712,11 +1735,11 @@ bool do_sync_pull(const std::vector<const char*>& srcs, const char* dst, bool co 
}



bool do_sync_sync(const std::string& lpath, const std::string& rpath, bool list_only,

                  CompressionType compression) {

                  CompressionType compression, bool dry_run) {

    SyncConnection sc;

    if (!sc.IsValid()) return false;



    bool success = copy_local_dir_remote(sc, lpath, rpath, true, list_only, compression);

    bool success = copy_local_dir_remote(sc, lpath, rpath, true, list_only, compression, dry_run);

    if (!list_only) {

        sc.ReportOverallTransferRate(TransferDirection::push);

    }

adb/client/file_sync_client.h

+2 −2
Original line number Diff line number Diff line
@@ -23,9 +23,9 @@ 


bool do_sync_ls(const char* path);

bool do_sync_push(const std::vector<const char*>& srcs, const char* dst, bool sync,

                  CompressionType compression);

                  CompressionType compression, bool dry_run);

bool do_sync_pull(const std::vector<const char*>& srcs, const char* dst, bool copy_attrs,

                  CompressionType compression, const char* name = nullptr);



bool do_sync_sync(const std::string& lpath, const std::string& rpath, bool list_only,

                  CompressionType compression);
 
                  CompressionType compression, bool dry_run);