Merge "logd: add a SerializedLogBuffer suitable for compression" (1e412e7d) · Commits · e / os / android_system_core

logd/Android.bp

+14 −1

Original line number	Diff line number	Diff line
		@@ -31,7 +31,11 @@ event_flag = [
		cc_defaults {
		name: "logd_defaults",

		shared_libs: ["libbase"],
		shared_libs: [
		"libbase",
		"libz",
		],
		static_libs: ["libzstd"],
		cflags: [
		"-Wextra",
		"-Wthread-safety",
		@@ -40,6 +44,7 @@ cc_defaults {
		lto: {
		thin: true,
		},
		cpp_std: "experimental",
		}

		cc_library_static {
		@@ -48,12 +53,16 @@ cc_library_static {
		host_supported: true,
		srcs: [
		"ChattyLogBuffer.cpp",
		"CompressionEngine.cpp",
		"LogReaderList.cpp",
		"LogReaderThread.cpp",
		"LogBufferElement.cpp",
		"LogStatistics.cpp",
		"LogWhiteBlackList.cpp",
		"LogTags.cpp",
		"SerializedFlushToState.cpp",
		"SerializedLogBuffer.cpp",
		"SerializedLogChunk.cpp",
		"SimpleLogBuffer.cpp",
		],
		logtags: ["event.logtags"],
		@@ -132,6 +141,8 @@ cc_defaults {
		"ChattyLogBufferTest.cpp",
		"logd_test.cpp",
		"LogBufferTest.cpp",
		"SerializedLogChunkTest.cpp",
		"SerializedFlushToStateTest.cpp",
		],

		static_libs: [
		@@ -140,6 +151,8 @@ cc_defaults {
		"liblog",
		"liblogd",
		"libselinux",
		"libz",
		"libzstd",
		],
		}

logd/CompressionEngine.cpp

0 → 100644

+104 −0

Original line number	Diff line number	Diff line
		/*
		* Copyright (C) 2020 The Android Open Source Project
		*
		* Licensed under the Apache License, Version 2.0 (the "License");
		* you may not use this file except in compliance with the License.
		* You may obtain a copy of the License at
		*
		* http://www.apache.org/licenses/LICENSE-2.0
		*
		* Unless required by applicable law or agreed to in writing, software
		* distributed under the License is distributed on an "AS IS" BASIS,
		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
		* See the License for the specific language governing permissions and
		* limitations under the License.
		*/

		#include "CompressionEngine.h"

		#include <limits>

		#include <android-base/logging.h>
		#include <zlib.h>
		#include <zstd.h>

		CompressionEngine& CompressionEngine::GetInstance() {
		CompressionEngine* engine = new ZstdCompressionEngine();
		return *engine;
		}

		bool ZlibCompressionEngine::Compress(std::span<uint8_t> in, std::vector<uint8_t>& out) {
		z_stream strm;
		strm.zalloc = Z_NULL;
		strm.zfree = Z_NULL;
		strm.opaque = Z_NULL;
		int ret = deflateInit(&strm, Z_DEFAULT_COMPRESSION);
		if (ret != Z_OK) {
		LOG(FATAL) << "deflateInit() failed";
		}

		CHECK_LE(in.size(), static_cast<int64_t>(std::numeric_limits<uint32_t>::max()));
		uint32_t out_size = deflateBound(&strm, in.size());

		out.resize(out_size);
		strm.avail_in = in.size();
		strm.next_in = const_cast<uint8_t*>(in.data());
		strm.avail_out = out_size;
		strm.next_out = out.data();
		ret = deflate(&strm, Z_FINISH);
		CHECK_EQ(ret, Z_STREAM_END);

		uint32_t compressed_data_size = strm.total_out;
		deflateEnd(&strm);
		out.resize(compressed_data_size);

		return true;
		}

		bool ZlibCompressionEngine::Decompress(const std::vector<uint8_t>& in, std::vector<uint8_t>& out,
		size_t out_size) {
		out.resize(out_size);

		z_stream strm;
		strm.zalloc = Z_NULL;
		strm.zfree = Z_NULL;
		strm.opaque = Z_NULL;
		strm.avail_in = in.size();
		strm.next_in = const_cast<uint8_t*>(in.data());
		strm.avail_out = out.size();
		strm.next_out = out.data();

		inflateInit(&strm);
		int ret = inflate(&strm, Z_NO_FLUSH);

		CHECK_EQ(strm.avail_in, 0U);
		CHECK_EQ(strm.avail_out, 0U);
		CHECK_EQ(ret, Z_STREAM_END);
		inflateEnd(&strm);

		return true;
		}

		bool ZstdCompressionEngine::Compress(std::span<uint8_t> in, std::vector<uint8_t>& out) {
		size_t out_size = ZSTD_compressBound(in.size());
		out.resize(out_size);

		out_size = ZSTD_compress(out.data(), out_size, in.data(), in.size(), 1);
		if (ZSTD_isError(out_size)) {
		LOG(FATAL) << "ZSTD_compress failed: " << ZSTD_getErrorName(out_size);
		}
		out.resize(out_size);

		return true;
		}

		bool ZstdCompressionEngine::Decompress(const std::vector<uint8_t>& in, std::vector<uint8_t>& out,
		size_t out_size) {
		out.resize(out_size);
		size_t result = ZSTD_decompress(out.data(), out.size(), in.data(), in.size());
		if (ZSTD_isError(result)) {
		LOG(FATAL) << "ZSTD_decompress failed: " << ZSTD_getErrorName(result);
		}
		CHECK_EQ(result, out.size());
		return true;
		}

logd/CompressionEngine.h

0 → 100644

+47 −0

Original line number	Diff line number	Diff line
		/*
		* Copyright (C) 2020 The Android Open Source Project
		*
		* Licensed under the Apache License, Version 2.0 (the "License");
		* you may not use this file except in compliance with the License.
		* You may obtain a copy of the License at
		*
		* http://www.apache.org/licenses/LICENSE-2.0
		*
		* Unless required by applicable law or agreed to in writing, software
		* distributed under the License is distributed on an "AS IS" BASIS,
		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
		* See the License for the specific language governing permissions and
		* limitations under the License.
		*/

		#pragma once

		#include <span>
		#include <vector>

		class CompressionEngine {
		public:
		static CompressionEngine& GetInstance();

		virtual ~CompressionEngine(){};

		virtual bool Compress(std::span<uint8_t> in, std::vector<uint8_t>& out) = 0;
		// Decompress the contents of `in` into `out`. `out_size` must be set to the decompressed size
		// of the contents.
		virtual bool Decompress(const std::vector<uint8_t>& in, std::vector<uint8_t>& out,
		size_t out_size) = 0;
		};

		class ZlibCompressionEngine : public CompressionEngine {
		public:
		bool Compress(std::span<uint8_t> in, std::vector<uint8_t>& out) override;
		bool Decompress(const std::vector<uint8_t>& in, std::vector<uint8_t>& out,
		size_t out_size) override;
		};

		class ZstdCompressionEngine : public CompressionEngine {
		public:
		bool Compress(std::span<uint8_t> in, std::vector<uint8_t>& out) override;
		bool Decompress(const std::vector<uint8_t>& in, std::vector<uint8_t>& out,
		size_t out_size) override;
		};
		No newline at end of file

logd/LogBufferTest.cpp

+2 −1

Original line number	Diff line number	Diff line
		@@ -455,4 +455,5 @@ TEST_P(LogBufferTest, clear_logs) {
		CompareLogMessages(after_clear_messages, read_log_messages_after_clear);
		}

		INSTANTIATE_TEST_CASE_P(LogBufferTests, LogBufferTest, testing::Values("chatty", "simple"));
		INSTANTIATE_TEST_CASE_P(LogBufferTests, LogBufferTest,
		testing::Values("chatty", "serialized", "simple"));

logd/LogBufferTest.h

+3 −0

Original line number	Diff line number	Diff line
		@@ -26,6 +26,7 @@
		#include "LogStatistics.h"
		#include "LogTags.h"
		#include "LogWhiteBlackList.h"
		#include "SerializedLogBuffer.h"
		#include "SimpleLogBuffer.h"

		struct LogMessage {
		@@ -67,6 +68,8 @@ class LogBufferTest : public testing::TestWithParam<std::string> {
		void SetUp() override {
		if (GetParam() == "chatty") {
		log_buffer_.reset(new ChattyLogBuffer(&reader_list_, &tags_, &prune_, &stats_));
		} else if (GetParam() == "serialized") {
		log_buffer_.reset(new SerializedLogBuffer(&reader_list_, &tags_, &stats_));
		} else if (GetParam() == "simple") {
		log_buffer_.reset(new SimpleLogBuffer(&reader_list_, &tags_, &stats_));
		} else {