Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit fb0c5cb2 authored by Josh Gao's avatar Josh Gao
Browse files

adb: improve display of transfer rate.

Show transfer rate for individual files, and for the overall transfer.

Bug: http://b/30411535
Change-Id: If2f3008f1287b1d1add4a23c3430b39885b6c904
Test: inspected output manually
parent 1a9979ec
Loading
Loading
Loading
Loading
+177 −78
Original line number Diff line number Diff line
@@ -28,8 +28,11 @@
#include <unistd.h>
#include <utime.h>

#include <chrono>
#include <functional>
#include <memory>
#include <sstream>
#include <string>
#include <vector>

#include "sysdeps.h"
@@ -94,14 +97,112 @@ struct copyinfo {
    }
};

enum class TransferDirection {
    push,
    pull,
};

struct TransferLedger {
    std::chrono::steady_clock::time_point start_time;
    uint64_t files_transferred;
    uint64_t files_skipped;
    uint64_t bytes_transferred;
    uint64_t bytes_expected;
    bool expect_multiple_files;

    TransferLedger() {
        Reset();
    }

    bool operator==(const TransferLedger& other) const {
        return files_transferred == other.files_transferred &&
               files_skipped == other.files_skipped && bytes_transferred == other.bytes_transferred;
    }

    bool operator!=(const TransferLedger& other) const {
        return !(*this == other);
    }

    void Reset() {
        start_time = std::chrono::steady_clock::now();
        files_transferred = 0;
        files_skipped = 0;
        bytes_transferred = 0;
        bytes_expected = 0;
    }

    std::string TransferRate() {
        if (bytes_transferred == 0) return "";

        std::chrono::duration<double> duration;
        duration = std::chrono::steady_clock::now() - start_time;

        double s = duration.count();
        if (s == 0) {
            return "";
        }
        double rate = (static_cast<double>(bytes_transferred) / s) / (1024 * 1024);
        return android::base::StringPrintf(" %.1f MB/s (%" PRIu64 " bytes in %.3fs)", rate,
                                           bytes_transferred, s);
    }

    void ReportProgress(LinePrinter& lp, const std::string& file, uint64_t file_copied_bytes,
                        uint64_t file_total_bytes) {
        char overall_percentage_str[5] = "?";
        if (bytes_expected != 0) {
            int overall_percentage = static_cast<int>(bytes_transferred * 100 / bytes_expected);
            // If we're pulling symbolic links, we'll pull the target of the link rather than
            // just create a local link, and that will cause us to go over 100%.
            if (overall_percentage <= 100) {
                snprintf(overall_percentage_str, sizeof(overall_percentage_str), "%d%%",
                         overall_percentage);
            }
        }

        std::string output;
        if (file_copied_bytes > file_total_bytes || file_total_bytes == 0) {
            // This case can happen if we're racing against something that wrote to the file
            // between our stat and our read, or if we're reading a magic file that lies about
            // its size. Just show how much we've copied.
            output = android::base::StringPrintf("[%4s] %s: %" PRId64 "/?", overall_percentage_str,
                                                 file.c_str(), file_copied_bytes);
        } else {
            // If we're transferring multiple files, we want to know how far through the current
            // file we are, as well as the overall percentage.
            if (expect_multiple_files) {
                int file_percentage = static_cast<int>(file_copied_bytes * 100 / file_total_bytes);
                output = android::base::StringPrintf("[%4s] %s: %d%%", overall_percentage_str,
                                                     file.c_str(), file_percentage);
            } else {
                output =
                    android::base::StringPrintf("[%4s] %s", overall_percentage_str, file.c_str());
            }
        }
        lp.Print(output, LinePrinter::LineType::INFO);
    }

    void ReportTransferRate(LinePrinter& lp, const std::string& name, TransferDirection direction) {
        const char* direction_str = (direction == TransferDirection::push) ? "pushed" : "pulled";
        std::stringstream ss;
        if (!name.empty()) {
            ss << name << ": ";
        }
        ss << files_transferred << " file" << ((files_transferred == 1) ? "" : "s") << " "
           << direction_str << ".";
        if (files_skipped > 0) {
            ss << " " << files_skipped << " file" << ((files_skipped == 1) ? "" : "s")
               << " skipped.";
        }
        ss << TransferRate();

        lp.Print(ss.str(), LinePrinter::LineType::INFO);
        lp.KeepInfoLine();
    }
};

class SyncConnection {
  public:
    SyncConnection()
            : total_bytes_(0),
              start_time_ms_(CurrentTimeMs()),
              expected_total_bytes_(0),
              expect_multiple_files_(false),
              expect_done_(false) {
    SyncConnection() : expect_done_(false) {
        max = SYNC_DATA_MAX; // TODO: decide at runtime.

        std::string error;
@@ -140,6 +241,40 @@ class SyncConnection {
        return rc != 0;
    }

    void NewTransfer() {
        current_ledger_.Reset();
    }

    void RecordBytesTransferred(size_t bytes) {
        current_ledger_.bytes_transferred += bytes;
        global_ledger_.bytes_transferred += bytes;
    }

    void RecordFilesTransferred(size_t files) {
        current_ledger_.files_transferred += files;
        global_ledger_.files_transferred += files;
    }

    void RecordFilesSkipped(size_t files) {
        current_ledger_.files_skipped += files;
        global_ledger_.files_skipped += files;
    }

    void ReportProgress(const std::string& file, uint64_t file_copied_bytes,
                        uint64_t file_total_bytes) {
        current_ledger_.ReportProgress(line_printer_, file, file_copied_bytes, file_total_bytes);
    }

    void ReportTransferRate(const std::string& file, TransferDirection direction) {
        current_ledger_.ReportTransferRate(line_printer_, file, direction);
    }

    void ReportOverallTransferRate(TransferDirection direction) {
        if (current_ledger_ != global_ledger_) {
            global_ledger_.ReportTransferRate(line_printer_, "", direction);
        }
    }

    bool SendRequest(int id, const char* path_and_mode) {
        size_t path_length = strlen(path_and_mode);
        if (path_length > 1024) {
@@ -199,7 +334,9 @@ class SyncConnection {

        WriteOrDie(lpath, rpath, &buf[0], (p - &buf[0]));
        expect_done_ = true;
        total_bytes_ += data_length;

        // RecordFilesTransferred gets called in CopyDone.
        RecordBytesTransferred(data_length);
        ReportProgress(rpath, data_length, data_length);
        return true;
    }
@@ -242,7 +379,7 @@ class SyncConnection {
            sbuf.size = bytes_read;
            WriteOrDie(lpath, rpath, &sbuf, sizeof(SyncRequest) + bytes_read);

            total_bytes_ += bytes_read;
            RecordBytesTransferred(bytes_read);
            bytes_copied += bytes_read;

            // Check to see if we've received an error from the other side.
@@ -259,6 +396,8 @@ class SyncConnection {
        msg.data.id = ID_DONE;
        msg.data.size = mtime;
        expect_done_ = true;

        // RecordFilesTransferred gets called in CopyDone.
        return WriteOrDie(lpath, rpath, &msg.data, sizeof(msg.data));
    }

@@ -271,6 +410,7 @@ class SyncConnection {
        if (msg.status.id == ID_OKAY) {
            if (expect_done_) {
                expect_done_ = false;
                RecordFilesTransferred(1);
                return true;
            } else {
                Error("failed to copy '%s' to '%s': received premature success", from, to);
@@ -296,44 +436,6 @@ class SyncConnection {
        return false;
    }

    std::string TransferRate() {
        uint64_t ms = CurrentTimeMs() - start_time_ms_;
        if (total_bytes_ == 0 || ms == 0) return "";

        double s = static_cast<double>(ms) / 1000LL;
        double rate = (static_cast<double>(total_bytes_) / s) / (1024*1024);
        return android::base::StringPrintf(" %.1f MB/s (%" PRId64 " bytes in %.3fs)",
                                           rate, total_bytes_, s);
    }

    void ReportProgress(const char* file, uint64_t file_copied_bytes, uint64_t file_total_bytes) {
        char overall_percentage_str[5] = "?";
        if (expected_total_bytes_ != 0) {
            int overall_percentage = static_cast<int>(total_bytes_ * 100 / expected_total_bytes_);
            // If we're pulling symbolic links, we'll pull the target of the link rather than
            // just create a local link, and that will cause us to go over 100%.
            if (overall_percentage <= 100) {
                snprintf(overall_percentage_str, sizeof(overall_percentage_str), "%d%%",
                         overall_percentage);
            }
        }

        if (file_copied_bytes > file_total_bytes || file_total_bytes == 0) {
            // This case can happen if we're racing against something that wrote to the file
            // between our stat and our read, or if we're reading a magic file that lies about
            // its size. Just show how much we've copied.
            Printf("[%4s] %s: %" PRId64 "/?", overall_percentage_str, file, file_copied_bytes);
        } else {
            // If we're transferring multiple files, we want to know how far through the current
            // file we are, as well as the overall percentage.
            if (expect_multiple_files_) {
                int file_percentage = static_cast<int>(file_copied_bytes * 100 / file_total_bytes);
                Printf("[%4s] %s: %d%%", overall_percentage_str, file, file_percentage);
            } else {
                Printf("[%4s] %s", overall_percentage_str, file);
            }
        }
    }

    void Printf(const char* fmt, ...) __attribute__((__format__(ADB_FORMAT_ARCHETYPE, 2, 3))) {
        std::string s;
@@ -381,33 +483,29 @@ class SyncConnection {
    }

    void ComputeExpectedTotalBytes(const std::vector<copyinfo>& file_list) {
        expected_total_bytes_ = 0;
        current_ledger_.bytes_expected = 0;
        for (const copyinfo& ci : file_list) {
            // Unfortunately, this doesn't work for symbolic links, because we'll copy the
            // target of the link rather than just creating a link. (But ci.size is the link size.)
            if (!ci.skip) expected_total_bytes_ += ci.size;
            if (!ci.skip) current_ledger_.bytes_expected += ci.size;
        }
        expect_multiple_files_ = true;
        current_ledger_.expect_multiple_files = true;
    }

    void SetExpectedTotalBytes(uint64_t expected_total_bytes) {
        expected_total_bytes_ = expected_total_bytes;
        expect_multiple_files_ = false;
        current_ledger_.bytes_expected = expected_total_bytes;
        current_ledger_.expect_multiple_files = false;
    }

    uint64_t total_bytes_;

    // TODO: add a char[max] buffer here, to replace syncsendbuf...
    int fd;
    size_t max;

  private:
    uint64_t start_time_ms_;

    uint64_t expected_total_bytes_;
    bool expect_multiple_files_;
    bool expect_done_;

    TransferLedger global_ledger_;
    TransferLedger current_ledger_;
    LinePrinter line_printer_;

    bool SendQuit() {
@@ -434,12 +532,6 @@ class SyncConnection {
        }
        return true;
    }

    static uint64_t CurrentTimeMs() {
        struct timeval tv;
        gettimeofday(&tv, 0); // (Not clock_gettime because of Mac/Windows.)
        return static_cast<uint64_t>(tv.tv_sec) * 1000 + tv.tv_usec / 1000;
    }
};

typedef void (sync_ls_cb)(unsigned mode, unsigned size, unsigned time, const char* name);
@@ -582,13 +674,12 @@ static bool sync_recv(SyncConnection& sc, const char* rpath, const char* lpath)
            return false;
        }

        sc.total_bytes_ += msg.data.size;

        bytes_copied += msg.data.size;

        sc.RecordBytesTransferred(msg.data.size);
        sc.ReportProgress(rpath, bytes_copied, size);
    }

    sc.RecordFilesTransferred(1);
    adb_close(lfd);
    return true;
}
@@ -673,13 +764,14 @@ static bool local_build_list(SyncConnection& sc, std::vector<copyinfo>* file_lis
static bool copy_local_dir_remote(SyncConnection& sc, std::string lpath,
                                  std::string rpath, bool check_timestamps,
                                  bool list_only) {
    sc.NewTransfer();

    // Make sure that both directory paths end in a slash.
    // Both paths are known to be nonempty, so we don't need to check.
    ensure_trailing_separators(lpath, rpath);

    // Recursively build the list of files to copy.
    std::vector<copyinfo> file_list;
    int pushed = 0;
    int skipped = 0;
    if (!local_build_list(sc, &file_list, lpath, rpath)) {
        return false;
@@ -717,15 +809,13 @@ static bool copy_local_dir_remote(SyncConnection& sc, std::string lpath,
                    return false;
                }
            }
            pushed++;
        } else {
            skipped++;
        }
    }

    sc.Println("%s: %d file%s pushed. %d file%s skipped.%s", rpath.c_str(), pushed,
               (pushed == 1) ? "" : "s", skipped, (skipped == 1) ? "" : "s",
               sc.TransferRate().c_str());
    sc.RecordFilesSkipped(skipped);
    sc.ReportTransferRate(lpath, TransferDirection::push);
    return true;
}

@@ -801,10 +891,14 @@ bool do_sync_push(const std::vector<const char*>& srcs, const char* dst) {
            path_holder += adb_basename(src_path);
            dst_path = path_holder.c_str();
        }

        sc.NewTransfer();
        sc.SetExpectedTotalBytes(st.st_size);
        success &= sync_send(sc, src_path, dst_path, st.st_mtime, st.st_mode);
        sc.ReportTransferRate(src_path, TransferDirection::push);
    }

    sc.ReportOverallTransferRate(TransferDirection::push);
    return success;
}

@@ -890,6 +984,8 @@ static int set_time_and_mode(const std::string& lpath, time_t time,

static bool copy_remote_dir_local(SyncConnection& sc, std::string rpath,
                                  std::string lpath, bool copy_attrs) {
    sc.NewTransfer();

    // Make sure that both directory paths end in a slash.
    // Both paths are known to be nonempty, so we don't need to check.
    ensure_trailing_separators(lpath, rpath);
@@ -903,7 +999,6 @@ static bool copy_remote_dir_local(SyncConnection& sc, std::string rpath,

    sc.ComputeExpectedTotalBytes(file_list);

    int pulled = 0;
    int skipped = 0;
    for (const copyinfo &ci : file_list) {
        if (!ci.skip) {
@@ -915,7 +1010,6 @@ static bool copy_remote_dir_local(SyncConnection& sc, std::string rpath,
                             ci.lpath.c_str(), strerror(errno));
                    return false;
                }
                pulled++;
                continue;
            }

@@ -926,15 +1020,13 @@ static bool copy_remote_dir_local(SyncConnection& sc, std::string rpath,
            if (copy_attrs && set_time_and_mode(ci.lpath, ci.time, ci.mode)) {
                return false;
            }
            pulled++;
        } else {
            skipped++;
        }
    }

    sc.Println("%s: %d file%s pulled. %d file%s skipped.%s", rpath.c_str(), pulled,
               (pulled == 1) ? "" : "s", skipped, (skipped == 1) ? "" : "s",
               sc.TransferRate().c_str());
    sc.RecordFilesSkipped(skipped);
    sc.ReportTransferRate(rpath, TransferDirection::pull);
    return true;
}

@@ -1033,6 +1125,7 @@ bool do_sync_pull(const std::vector<const char*>& srcs, const char* dst,
            dst_path = path_holder.c_str();
        }

        sc.NewTransfer();
        sc.SetExpectedTotalBytes(src_size);
        if (!sync_recv(sc, src_path, dst_path)) {
            success = false;
@@ -1043,8 +1136,10 @@ bool do_sync_pull(const std::vector<const char*>& srcs, const char* dst,
            success = false;
            continue;
        }
        sc.ReportTransferRate(src_path, TransferDirection::pull);
    }

    sc.ReportOverallTransferRate(TransferDirection::pull);
    return success;
}

@@ -1052,5 +1147,9 @@ bool do_sync_sync(const std::string& lpath, const std::string& rpath, bool list_
    SyncConnection sc;
    if (!sc.IsValid()) return false;

    return copy_local_dir_remote(sc, lpath, rpath, true, list_only);
    bool success = copy_local_dir_remote(sc, lpath, rpath, true, list_only);
    if (!list_only) {
        sc.ReportOverallTransferRate(TransferDirection::push);
    }
    return success;
}