Merge "adb: implement compression for file sync." (4dc73c0b) · Commits · e / os / android_system_core

adb/Android.bp

+4 −0

Original line number	Diff line number	Diff line
		@@ -317,6 +317,7 @@ cc_binary_host {
		"libandroidfw",
		"libapp_processes_protos_full",
		"libbase",
		"libbrotli",
		"libcutils",
		"libcrypto_utils",
		"libcrypto",
		@@ -469,6 +470,7 @@ cc_library {
		static_libs: [
		"libadbconnection_server",
		"libadbd_core",
		"libbrotli",
		"libdiagnose_usb",
		],

		@@ -567,6 +569,7 @@ cc_library {
		},

		static_libs: [
		"libbrotli",
		"libcutils_sockets",
		"libdiagnose_usb",
		"libmdnssd",
		@@ -606,6 +609,7 @@ cc_binary {
		"libapp_processes_protos_lite",
		"libasyncio",
		"libbase",
		"libbrotli",
		"libcap",
		"libcrypto_utils",
		"libcutils_sockets",

adb/brotli_utils.h

0 → 100644

+144 −0

Original line number	Diff line number	Diff line
		/*
		* Copyright (C) 2020 The Android Open Source Project
		*
		* Licensed under the Apache License, Version 2.0 (the "License");
		* you may not use this file except in compliance with the License.
		* You may obtain a copy of the License at
		*
		* http://www.apache.org/licenses/LICENSE-2.0
		*
		* Unless required by applicable law or agreed to in writing, software
		* distributed under the License is distributed on an "AS IS" BASIS,
		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
		* See the License for the specific language governing permissions and
		* limitations under the License.
		*/

		#pragma once

		#include <span>

		#include <brotli/decode.h>
		#include <brotli/encode.h>

		#include "types.h"

		enum class BrotliDecodeResult {
		Error,
		Done,
		NeedInput,
		MoreOutput,
		};

		struct BrotliDecoder {
		explicit BrotliDecoder(std::span<char> output_buffer)
		: output_buffer_(output_buffer),
		decoder_(BrotliDecoderCreateInstance(nullptr, nullptr, nullptr),
		BrotliDecoderDestroyInstance) {}

		void Append(Block&& block) { input_buffer_.append(std::move(block)); }

		BrotliDecodeResult Decode(std::span<char>* output) {
		size_t available_in = input_buffer_.front_size();
		const uint8_t* next_in = reinterpret_cast<const uint8_t*>(input_buffer_.front_data());

		size_t available_out = output_buffer_.size();
		uint8_t* next_out = reinterpret_cast<uint8_t*>(output_buffer_.data());

		BrotliDecoderResult r = BrotliDecoderDecompressStream(
		decoder_.get(), &available_in, &next_in, &available_out, &next_out, nullptr);

		size_t bytes_consumed = input_buffer_.front_size() - available_in;
		input_buffer_.drop_front(bytes_consumed);

		size_t bytes_emitted = output_buffer_.size() - available_out;
		*output = std::span<char>(output_buffer_.data(), bytes_emitted);

		switch (r) {
		case BROTLI_DECODER_RESULT_SUCCESS:
		return BrotliDecodeResult::Done;
		case BROTLI_DECODER_RESULT_ERROR:
		return BrotliDecodeResult::Error;
		case BROTLI_DECODER_RESULT_NEEDS_MORE_INPUT:
		// Brotli guarantees as one of its invariants that if it returns NEEDS_MORE_INPUT,
		// it will consume the entire input buffer passed in, so we don't have to worry
		// about bytes left over in the front block with more input remaining.
		return BrotliDecodeResult::NeedInput;
		case BROTLI_DECODER_RESULT_NEEDS_MORE_OUTPUT:
		return BrotliDecodeResult::MoreOutput;
		}
		}

		private:
		IOVector input_buffer_;
		std::span<char> output_buffer_;
		std::unique_ptr<BrotliDecoderState, void ()(BrotliDecoderState)> decoder_;
		};

		enum class BrotliEncodeResult {
		Error,
		Done,
		NeedInput,
		MoreOutput,
		};

		template <size_t OutputBlockSize>
		struct BrotliEncoder {
		explicit BrotliEncoder()
		: output_block_(OutputBlockSize),
		output_bytes_left_(OutputBlockSize),
		encoder_(BrotliEncoderCreateInstance(nullptr, nullptr, nullptr),
		BrotliEncoderDestroyInstance) {
		BrotliEncoderSetParameter(encoder_.get(), BROTLI_PARAM_QUALITY, 1);
		}

		void Append(Block input) { input_buffer_.append(std::move(input)); }
		void Finish() { finished_ = true; }

		BrotliEncodeResult Encode(Block* output) {
		output->clear();
		while (true) {
		size_t available_in = input_buffer_.front_size();
		const uint8_t* next_in = reinterpret_cast<const uint8_t*>(input_buffer_.front_data());

		size_t available_out = output_bytes_left_;
		uint8_t* next_out = reinterpret_cast<uint8_t*>(output_block_.data() +
		(OutputBlockSize - output_bytes_left_));

		BrotliEncoderOperation op = BROTLI_OPERATION_PROCESS;
		if (finished_) {
		op = BROTLI_OPERATION_FINISH;
		}

		if (!BrotliEncoderCompressStream(encoder_.get(), op, &available_in, &next_in,
		&available_out, &next_out, nullptr)) {
		return BrotliEncodeResult::Error;
		}

		size_t bytes_consumed = input_buffer_.front_size() - available_in;
		input_buffer_.drop_front(bytes_consumed);

		output_bytes_left_ = available_out;

		if (BrotliEncoderIsFinished(encoder_.get())) {
		output_block_.resize(OutputBlockSize - output_bytes_left_);
		*output = std::move(output_block_);
		return BrotliEncodeResult::Done;
		} else if (output_bytes_left_ == 0) {
		*output = std::move(output_block_);
		output_block_.resize(OutputBlockSize);
		output_bytes_left_ = OutputBlockSize;
		return BrotliEncodeResult::MoreOutput;
		} else if (input_buffer_.empty()) {
		return BrotliEncodeResult::NeedInput;
		}
		}
		}

		private:
		bool finished_ = false;
		IOVector input_buffer_;
		Block output_block_;
		size_t output_bytes_left_;
		std::unique_ptr<BrotliEncoderState, void ()(BrotliEncoderState)> encoder_;
		};

adb/client/adb_install.cpp

+1 −1

Original line number	Diff line number	Diff line
		@@ -286,7 +286,7 @@ static int install_app_legacy(int argc, const char** argv, bool use_fastdeploy)
		}
		}

		if (do_sync_push(apk_file, apk_dest.c_str(), false)) {
		if (do_sync_push(apk_file, apk_dest.c_str(), false, true)) {
		result = pm_command(argc, argv);
		delete_device_file(apk_dest);
		}

adb/client/commandline.cpp

+58 −18

Original line number	Diff line number	Diff line
		@@ -129,15 +129,21 @@ static void help() {
		" reverse --remove-all remove all reverse socket connections from device\n"
		"\n"
		"file transfer:\n"
		" push [--sync] LOCAL... REMOTE\n"
		" push [--sync] [-zZ] LOCAL... REMOTE\n"
		" copy local files/directories to device\n"
		" --sync: only push files that are newer on the host than the device\n"
		" pull [-a] REMOTE... LOCAL\n"
		" -z: enable compression\n"
		" -Z: disable compression\n"
		" pull [-azZ] REMOTE... LOCAL\n"
		" copy files/dirs from device\n"
		" -a: preserve file timestamp and mode\n"
		" sync [all\|data\|odm\|oem\|product\|system\|system_ext\|vendor]\n"
		" -z: enable compression\n"
		" -Z: disable compression\n"
		" sync [-lzZ] [all\|data\|odm\|oem\|product\|system\|system_ext\|vendor]\n"
		" sync a local build from $ANDROID_PRODUCT_OUT to the device (default all)\n"
		" -l: list files that would be copied, but don't copy them\n"
		" -z: enable compression\n"
		" -Z: disable compression\n"
		"\n"
		"shell:\n"
		" shell [-e ESCAPE] [-n] [-Tt] [-x] [COMMAND...]\n"
		@@ -1309,8 +1315,12 @@ static int restore(int argc, const char** argv) {
		}

		static void parse_push_pull_args(const char** arg, int narg, std::vector<const char> srcs,
		const char** dst, bool* copy_attrs, bool* sync) {
		const char** dst, bool* copy_attrs, bool* sync, bool* compressed) {
		*copy_attrs = false;
		const char* adb_compression = getenv("ADB_COMPRESSION");
		if (adb_compression && strcmp(adb_compression, "0") == 0) {
		*compressed = false;
		}

		srcs->clear();
		bool ignore_flags = false;
		@@ -1322,6 +1332,14 @@ static void parse_push_pull_args(const char** arg, int narg, std::vector<const c
		// Silently ignore for backwards compatibility.
		} else if (!strcmp(*arg, "-a")) {
		*copy_attrs = true;
		} else if (!strcmp(*arg, "-z")) {
		if (compressed != nullptr) {
		*compressed = true;
		}
		} else if (!strcmp(*arg, "-Z")) {
		if (compressed != nullptr) {
		*compressed = false;
		}
		} else if (!strcmp(*arg, "--sync")) {
		if (sync != nullptr) {
		*sync = true;
		@@ -1876,20 +1894,22 @@ int adb_commandline(int argc, const char** argv) {
		} else if (!strcmp(argv[0], "push")) {
		bool copy_attrs = false;
		bool sync = false;
		bool compressed = true;
		std::vector<const char*> srcs;
		const char* dst = nullptr;

		parse_push_pull_args(&argv[1], argc - 1, &srcs, &dst, &copy_attrs, &sync);
		parse_push_pull_args(&argv[1], argc - 1, &srcs, &dst, &copy_attrs, &sync, &compressed);
		if (srcs.empty() \|\| !dst) error_exit("push requires an argument");
		return do_sync_push(srcs, dst, sync) ? 0 : 1;
		return do_sync_push(srcs, dst, sync, compressed) ? 0 : 1;
		} else if (!strcmp(argv[0], "pull")) {
		bool copy_attrs = false;
		bool compressed = true;
		std::vector<const char*> srcs;
		const char* dst = ".";

		parse_push_pull_args(&argv[1], argc - 1, &srcs, &dst, &copy_attrs, nullptr);
		parse_push_pull_args(&argv[1], argc - 1, &srcs, &dst, &copy_attrs, nullptr, &compressed);
		if (srcs.empty()) error_exit("pull requires an argument");
		return do_sync_pull(srcs, dst, copy_attrs) ? 0 : 1;
		return do_sync_pull(srcs, dst, copy_attrs, compressed) ? 0 : 1;
		} else if (!strcmp(argv[0], "install")) {
		if (argc < 2) error_exit("install requires an argument");
		return install_app(argc, argv);
		@@ -1905,18 +1925,38 @@ int adb_commandline(int argc, const char** argv) {
		} else if (!strcmp(argv[0], "sync")) {
		std::string src;
		bool list_only = false;
		if (argc < 2) {
		// No partition specified: sync all of them.
		} else if (argc >= 2 && strcmp(argv[1], "-l") == 0) {
		bool compressed = true;

		const char* adb_compression = getenv("ADB_COMPRESSION");
		if (adb_compression && strcmp(adb_compression, "0") == 0) {
		compressed = false;
		}

		int opt;
		while ((opt = getopt(argc, const_cast<char**>(argv), "lzZ")) != -1) {
		switch (opt) {
		case 'l':
		list_only = true;
		if (argc == 3) src = argv[2];
		} else if (argc == 2) {
		src = argv[1];
		break;
		case 'z':
		compressed = true;
		break;
		case 'Z':
		compressed = false;
		break;
		default:
		error_exit("usage: adb sync [-lzZ] [PARTITION]");
		}
		}

		if (optind == argc) {
		src = "all";
		} else if (optind + 1 == argc) {
		src = argv[optind];
		} else {
		error_exit("usage: adb sync [-l] [PARTITION]");
		error_exit("usage: adb sync [-lzZ] [PARTITION]");
		}

		if (src.empty()) src = "all";
		std::vector<std::string> partitions{"data", "odm", "oem", "product",
		"system", "system_ext", "vendor"};
		bool found = false;
		@@ -1925,7 +1965,7 @@ int adb_commandline(int argc, const char** argv) {
		std::string src_dir{product_file(partition)};
		if (!directory_exists(src_dir)) continue;
		found = true;
		if (!do_sync_sync(src_dir, "/" + partition, list_only)) return 1;
		if (!do_sync_sync(src_dir, "/" + partition, list_only, compressed)) return 1;
		}
		}
		if (!found) error_exit("don't know how to sync %s partition", src.c_str());

adb/client/fastdeploy.cpp

+1 −1

Original line number	Diff line number	Diff line
		@@ -112,7 +112,7 @@ static void push_to_device(const void* data, size_t byte_count, const char* dst,
		// but can't be removed until after the push.
		unix_close(tf.release());

		if (!do_sync_push(srcs, dst, sync)) {
		if (!do_sync_push(srcs, dst, sync, true)) {
		error_exit("Failed to push fastdeploy agent to device.");
		}
		}