Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 1a796bca authored by Tom Cherry's avatar Tom Cherry
Browse files

logd: add a SerializedLogBuffer suitable for compression 

Initial commit for a SerializedLogBuffer.  The intention here is for
the serialized data to be compressed (currently using zlib) to allow
for substantially longer logs in the same memory footprint.

Test: unit tests
Change-Id: I2528e4e1ff1cf3bc91130173a107f371f04d911a
parent 8d326553

logd/Android.bp

+14 −1
Original line number Diff line number Diff line
@@ -31,7 +31,11 @@ event_flag = [ 
cc_defaults {

    name: "logd_defaults",



    shared_libs: ["libbase"],

    shared_libs: [

        "libbase",

        "libz",

    ],

    static_libs: ["libzstd"],

    cflags: [

        "-Wextra",

        "-Wthread-safety",
@@ -40,6 +44,7 @@ cc_defaults { 
    lto: {

        thin: true,

    },

    cpp_std: "experimental",

}



cc_library_static {
@@ -48,12 +53,16 @@ cc_library_static { 
    host_supported: true,

    srcs: [

        "ChattyLogBuffer.cpp",

        "CompressionEngine.cpp",

        "LogReaderList.cpp",

        "LogReaderThread.cpp",

        "LogBufferElement.cpp",

        "LogStatistics.cpp",

        "LogWhiteBlackList.cpp",

        "LogTags.cpp",

        "SerializedFlushToState.cpp",

        "SerializedLogBuffer.cpp",

        "SerializedLogChunk.cpp",

        "SimpleLogBuffer.cpp",

    ],

    logtags: ["event.logtags"],
@@ -132,6 +141,8 @@ cc_defaults { 
        "ChattyLogBufferTest.cpp",

        "logd_test.cpp",

        "LogBufferTest.cpp",

        "SerializedLogChunkTest.cpp",

        "SerializedFlushToStateTest.cpp",

    ],



    static_libs: [
@@ -140,6 +151,8 @@ cc_defaults { 
        "liblog",

        "liblogd",

        "libselinux",

        "libz",

        "libzstd",

    ],

}

logd/CompressionEngine.cpp

0 → 100644
+104 −0
Original line number Diff line number Diff line 
/*

 * Copyright (C) 2020 The Android Open Source Project

 *

 * Licensed under the Apache License, Version 2.0 (the "License");

 * you may not use this file except in compliance with the License.

 * You may obtain a copy of the License at

 *

 *      http://www.apache.org/licenses/LICENSE-2.0

 *

 * Unless required by applicable law or agreed to in writing, software

 * distributed under the License is distributed on an "AS IS" BASIS,

 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

 * See the License for the specific language governing permissions and

 * limitations under the License.

 */



#include "CompressionEngine.h"



#include <limits>



#include <android-base/logging.h>

#include <zlib.h>

#include <zstd.h>



CompressionEngine& CompressionEngine::GetInstance() {

    CompressionEngine* engine = new ZstdCompressionEngine();

    return *engine;

}



bool ZlibCompressionEngine::Compress(std::span<uint8_t> in, std::vector<uint8_t>& out) {

    z_stream strm;

    strm.zalloc = Z_NULL;

    strm.zfree = Z_NULL;

    strm.opaque = Z_NULL;

    int ret = deflateInit(&strm, Z_DEFAULT_COMPRESSION);

    if (ret != Z_OK) {

        LOG(FATAL) << "deflateInit() failed";

    }



    CHECK_LE(in.size(), static_cast<int64_t>(std::numeric_limits<uint32_t>::max()));

    uint32_t out_size = deflateBound(&strm, in.size());



    out.resize(out_size);

    strm.avail_in = in.size();

    strm.next_in = const_cast<uint8_t*>(in.data());

    strm.avail_out = out_size;

    strm.next_out = out.data();

    ret = deflate(&strm, Z_FINISH);

    CHECK_EQ(ret, Z_STREAM_END);



    uint32_t compressed_data_size = strm.total_out;

    deflateEnd(&strm);

    out.resize(compressed_data_size);



    return true;

}



bool ZlibCompressionEngine::Decompress(const std::vector<uint8_t>& in, std::vector<uint8_t>& out,

                                       size_t out_size) {

    out.resize(out_size);



    z_stream strm;

    strm.zalloc = Z_NULL;

    strm.zfree = Z_NULL;

    strm.opaque = Z_NULL;

    strm.avail_in = in.size();

    strm.next_in = const_cast<uint8_t*>(in.data());

    strm.avail_out = out.size();

    strm.next_out = out.data();



    inflateInit(&strm);

    int ret = inflate(&strm, Z_NO_FLUSH);



    CHECK_EQ(strm.avail_in, 0U);

    CHECK_EQ(strm.avail_out, 0U);

    CHECK_EQ(ret, Z_STREAM_END);

    inflateEnd(&strm);



    return true;

}



bool ZstdCompressionEngine::Compress(std::span<uint8_t> in, std::vector<uint8_t>& out) {

    size_t out_size = ZSTD_compressBound(in.size());

    out.resize(out_size);



    out_size = ZSTD_compress(out.data(), out_size, in.data(), in.size(), 1);

    if (ZSTD_isError(out_size)) {

        LOG(FATAL) << "ZSTD_compress failed: " << ZSTD_getErrorName(out_size);

    }

    out.resize(out_size);



    return true;

}



bool ZstdCompressionEngine::Decompress(const std::vector<uint8_t>& in, std::vector<uint8_t>& out,

                                       size_t out_size) {

    out.resize(out_size);

    size_t result = ZSTD_decompress(out.data(), out.size(), in.data(), in.size());

    if (ZSTD_isError(result)) {

        LOG(FATAL) << "ZSTD_decompress failed: " << ZSTD_getErrorName(result);

    }

    CHECK_EQ(result, out.size());

    return true;

}

logd/CompressionEngine.h

0 → 100644
+47 −0
Original line number Diff line number Diff line 
/*

 * Copyright (C) 2020 The Android Open Source Project

 *

 * Licensed under the Apache License, Version 2.0 (the "License");

 * you may not use this file except in compliance with the License.

 * You may obtain a copy of the License at

 *

 *      http://www.apache.org/licenses/LICENSE-2.0

 *

 * Unless required by applicable law or agreed to in writing, software

 * distributed under the License is distributed on an "AS IS" BASIS,

 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

 * See the License for the specific language governing permissions and

 * limitations under the License.

 */



#pragma once



#include <span>

#include <vector>



class CompressionEngine {

  public:

    static CompressionEngine& GetInstance();



    virtual ~CompressionEngine(){};



    virtual bool Compress(std::span<uint8_t> in, std::vector<uint8_t>& out) = 0;

    // Decompress the contents of `in` into `out`.  `out_size` must be set to the decompressed size

    // of the contents.

    virtual bool Decompress(const std::vector<uint8_t>& in, std::vector<uint8_t>& out,

                            size_t out_size) = 0;

};



class ZlibCompressionEngine : public CompressionEngine {

  public:

    bool Compress(std::span<uint8_t> in, std::vector<uint8_t>& out) override;

    bool Decompress(const std::vector<uint8_t>& in, std::vector<uint8_t>& out,

                    size_t out_size) override;

};



class ZstdCompressionEngine : public CompressionEngine {

  public:

    bool Compress(std::span<uint8_t> in, std::vector<uint8_t>& out) override;

    bool Decompress(const std::vector<uint8_t>& in, std::vector<uint8_t>& out,

                    size_t out_size) override;

};
 
 No newline at end of file

logd/LogBufferTest.cpp

+2 −1
Original line number Diff line number Diff line
@@ -455,4 +455,5 @@ TEST_P(LogBufferTest, clear_logs) { 
    CompareLogMessages(after_clear_messages, read_log_messages_after_clear);

}



INSTANTIATE_TEST_CASE_P(LogBufferTests, LogBufferTest, testing::Values("chatty", "simple"));
 
INSTANTIATE_TEST_CASE_P(LogBufferTests, LogBufferTest,

                        testing::Values("chatty", "serialized", "simple"));

logd/LogBufferTest.h

+3 −0
Original line number Diff line number Diff line
@@ -26,6 +26,7 @@ 
#include "LogStatistics.h"

#include "LogTags.h"

#include "LogWhiteBlackList.h"

#include "SerializedLogBuffer.h"

#include "SimpleLogBuffer.h"



struct LogMessage {
@@ -67,6 +68,8 @@ class LogBufferTest : public testing::TestWithParam<std::string> { 
    void SetUp() override {

        if (GetParam() == "chatty") {

            log_buffer_.reset(new ChattyLogBuffer(&reader_list_, &tags_, &prune_, &stats_));

        } else if (GetParam() == "serialized") {

            log_buffer_.reset(new SerializedLogBuffer(&reader_list_, &tags_, &stats_));

        } else if (GetParam() == "simple") {

            log_buffer_.reset(new SimpleLogBuffer(&reader_list_, &tags_, &stats_));

        } else {