Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 63fe86a4 authored by Mike Ma's avatar Mike Ma Committed by Automerger Merge Worker
Browse files

Merge "Optimize memory usage in incidentd" into rvc-dev am: 06b3aaee 

Change-Id: Ib0a0e5a54d2f060e15abc7f1f452a79152d8f731
parents 802cc062 06b3aaee

cmds/incidentd/src/FdBuffer.cpp

+15 −7
Original line number Diff line number Diff line
@@ -17,6 +17,7 @@ 
#include "Log.h"



#include "FdBuffer.h"

#include "incidentd_util.h"



#include <log/log.h>

#include <utils/SystemClock.h>
@@ -31,17 +32,24 @@ namespace os { 
namespace incidentd {



const ssize_t BUFFER_SIZE = 16 * 1024;  // 16 KB

const ssize_t MAX_BUFFER_COUNT = 6144;   // 96 MB max

const ssize_t MAX_BUFFER_SIZE = 96 * 1024 * 1024;  // 96 MB



FdBuffer::FdBuffer()

        :mBuffer(new EncodedBuffer(BUFFER_SIZE)),

FdBuffer::FdBuffer(): FdBuffer(get_buffer_from_pool(), /* isBufferPooled= */ true)  {

}



FdBuffer::FdBuffer(sp<EncodedBuffer> buffer, bool isBufferPooled)

        :mBuffer(buffer),

         mStartTime(-1),

         mFinishTime(-1),

         mTimedOut(false),

         mTruncated(false) {

         mTruncated(false),

         mIsBufferPooled(isBufferPooled) {

}



FdBuffer::~FdBuffer() {

    if (mIsBufferPooled) {

        return_buffer_to_pool(mBuffer);

    }

}



status_t FdBuffer::read(int fd, int64_t timeout) {
@@ -51,7 +59,7 @@ status_t FdBuffer::read(int fd, int64_t timeout) { 
    fcntl(fd, F_SETFL, fcntl(fd, F_GETFL, 0) | O_NONBLOCK);



    while (true) {

        if (mBuffer->size() >= MAX_BUFFER_COUNT * BUFFER_SIZE) {

        if (mBuffer->size() >= MAX_BUFFER_SIZE) {

            mTruncated = true;

            VLOG("Truncating data");

            break;
@@ -106,7 +114,7 @@ status_t FdBuffer::readFully(int fd) { 
    mStartTime = uptimeMillis();



    while (true) {

        if (mBuffer->size() >= MAX_BUFFER_COUNT * BUFFER_SIZE) {

        if (mBuffer->size() >= MAX_BUFFER_SIZE) {

            // Don't let it get too big.

            mTruncated = true;

            VLOG("Truncating data");
@@ -156,7 +164,7 @@ status_t FdBuffer::readProcessedDataInStream(int fd, unique_fd toFd, unique_fd f 


    // This is the buffer used to store processed data

    while (true) {

        if (mBuffer->size() >= MAX_BUFFER_COUNT * BUFFER_SIZE) {

        if (mBuffer->size() >= MAX_BUFFER_SIZE) {

            VLOG("Truncating data");

            mTruncated = true;

            break;

cmds/incidentd/src/FdBuffer.h

+2 −0
Original line number Diff line number Diff line
@@ -35,6 +35,7 @@ using namespace android::util; 
class FdBuffer {

public:

    FdBuffer();

    FdBuffer(sp<EncodedBuffer> buffer, bool isBufferPooled = false);

    ~FdBuffer();



    /**
@@ -114,6 +115,7 @@ private: 
    int64_t mFinishTime;

    bool mTimedOut;

    bool mTruncated;

    bool mIsBufferPooled;

};



}  // namespace incidentd

cmds/incidentd/src/PrivacyFilter.cpp

+13 −7
Original line number Diff line number Diff line
@@ -19,9 +19,6 @@ 
#include "incidentd_util.h"

#include "PrivacyFilter.h"

#include "proto_util.h"



#include "incidentd_util.h"

#include "proto_util.h"

#include "Section.h"



#include <android-base/file.h>
@@ -129,6 +126,8 @@ public: 
    FieldStripper(const Privacy* restrictions, const sp<ProtoReader>& data,

            uint8_t bufferLevel);



    ~FieldStripper();



    /**

     * Take the data that we have, and filter it down so that no fields

     * are more sensitive than the given privacy policy.
@@ -167,6 +166,7 @@ private: 
     */

    uint8_t mCurrentLevel;



    sp<EncodedBuffer> mEncodedBuffer;

};



FieldStripper::FieldStripper(const Privacy* restrictions, const sp<ProtoReader>& data,
@@ -174,19 +174,25 @@ FieldStripper::FieldStripper(const Privacy* restrictions, const sp<ProtoReader>& 
        :mRestrictions(restrictions),

         mData(data),

         mSize(data->size()),

         mCurrentLevel(bufferLevel) {

         mCurrentLevel(bufferLevel),

         mEncodedBuffer(get_buffer_from_pool()) {

    if (mSize < 0) {

        ALOGW("FieldStripper constructed with a ProtoReader that doesn't support size."

                " Data will be missing.");

    }

}



FieldStripper::~FieldStripper() {

    return_buffer_to_pool(mEncodedBuffer);

}



status_t FieldStripper::strip(const uint8_t privacyPolicy) {

    // If the current strip level is less (fewer fields retained) than what's already in the

    // buffer, then we can skip it.

    if (mCurrentLevel < privacyPolicy) {

        PrivacySpec spec(privacyPolicy);

        ProtoOutputStream proto;

        mEncodedBuffer->clear();

        ProtoOutputStream proto(mEncodedBuffer);



        // Optimization when no strip happens.

        if (mRestrictions == NULL || spec.RequireAll()) {
@@ -370,7 +376,7 @@ status_t filter_and_write_report(int to, int from, uint8_t bufferLevel, 
            write_field_or_skip(NULL, reader, fieldTag, true);

        }

    }



    clear_buffer_pool();

    err = reader->getError();

    if (err != NO_ERROR) {

        ALOGW("filter_and_write_report reader had an error: %s", strerror(-err));

cmds/incidentd/src/Reporter.cpp

+1 −1
Original line number Diff line number Diff line
@@ -698,7 +698,7 @@ DONE: 
            listener->onReportFailed();

        });

    }



    clear_buffer_pool();

    ALOGI("Done taking incident report err=%s", strerror(-err));

}

cmds/incidentd/src/Section.cpp

+77 −38
Original line number Diff line number Diff line
@@ -36,6 +36,7 @@ 
#include <log/log_read.h>

#include <log/logprint.h>

#include <private/android_logger.h>

#include <sys/mman.h>



#include "FdBuffer.h"

#include "Privacy.h"
@@ -106,7 +107,6 @@ status_t FileSection::Execute(ReportWriter* writer) const { 
        return NO_ERROR;

    }



    FdBuffer buffer;

    Fpipe p2cPipe;

    Fpipe c2pPipe;

    // initiate pipes to pass data to/from incident_helper
@@ -122,6 +122,7 @@ status_t FileSection::Execute(ReportWriter* writer) const { 
    }



    // parent process

    FdBuffer buffer;

    status_t readStatus = buffer.readProcessedDataInStream(fd.get(), std::move(p2cPipe.writeFd()),

                                                           std::move(c2pPipe.readFd()),

                                                           this->timeoutMs, mIsSysfs);
@@ -356,7 +357,6 @@ CommandSection::CommandSection(int id, const char* command, ...) : Section(id) { 
CommandSection::~CommandSection() { free(mCommand); }



status_t CommandSection::Execute(ReportWriter* writer) const {

    FdBuffer buffer;

    Fpipe cmdPipe;

    Fpipe ihPipe;
@@ -377,6 +377,7 @@ status_t CommandSection::Execute(ReportWriter* writer) const { 
    }



    cmdPipe.writeFd().reset();

    FdBuffer buffer;

    status_t readStatus = buffer.read(ihPipe.readFd().get(), this->timeoutMs);

    writer->setSectionStats(buffer);

    if (readStatus != NO_ERROR || buffer.timedOut()) {
@@ -574,6 +575,16 @@ static inline int32_t get4LE(uint8_t const* src) { 
}



status_t LogSection::BlockingCall(unique_fd& pipeWriteFd) const {

    // heap profile shows that liblog malloc & free significant amount of memory in this process.

    // Hence forking a new process to prevent memory fragmentation.

    pid_t pid = fork();

    if (pid < 0) {

        ALOGW("[%s] failed to fork", this->name.string());

        return errno;

    }

    if (pid > 0) {

        return wait_child(pid, this->timeoutMs);

    }

    // Open log buffer and getting logs since last retrieved time if any.

    unique_ptr<logger_list, void (*)(logger_list*)> loggers(

            gLastLogsRetrieved.find(mLogID) == gLastLogsRetrieved.end()
@@ -583,31 +594,31 @@ status_t LogSection::BlockingCall(unique_fd& pipeWriteFd) const { 


    if (android_logger_open(loggers.get(), mLogID) == NULL) {

        ALOGE("[%s] Can't get logger.", this->name.string());

        return -1;

        _exit(EXIT_FAILURE);

    }



    log_msg msg;

    log_time lastTimestamp(0);



    ProtoOutputStream proto;

    status_t err = OK;

    while (true) {  // keeps reading until logd buffer is fully read.

        status_t err = android_logger_list_read(loggers.get(), &msg);

        // err = 0 - no content, unexpected connection drop or EOF.

        // err = +ive number - size of retrieved data from logger

        // err = -ive number, OS supplied error _except_ for -EAGAIN

        // err = -EAGAIN, graceful indication for ANDRODI_LOG_NONBLOCK that this is the end of data.

        if (err <= 0) {

            if (err != -EAGAIN) {

        status_t status = android_logger_list_read(loggers.get(), &msg);

        // status = 0 - no content, unexpected connection drop or EOF.

        // status = +ive number - size of retrieved data from logger

        // status = -ive number, OS supplied error _except_ for -EAGAIN

        // status = -EAGAIN, graceful indication for ANDRODI_LOG_NONBLOCK that this is the end.

        if (status <= 0) {

            if (status != -EAGAIN) {

                ALOGW("[%s] fails to read a log_msg.\n", this->name.string());

                err = -status;

            }

            // dump previous logs and don't consider this error a failure.

            break;

        }

        if (mBinary) {

            // remove the first uint32 which is tag's index in event log tags

            android_log_context context = create_android_log_parser(msg.msg() + sizeof(uint32_t),

                                                                    msg.len() - sizeof(uint32_t));

            ;

            android_log_list_element elem;



            lastTimestamp.tv_sec = msg.entry.sec;
@@ -667,9 +678,10 @@ status_t LogSection::BlockingCall(unique_fd& pipeWriteFd) const { 
            }

        } else {

            AndroidLogEntry entry;

            err = android_log_processLogBuffer(&msg.entry, &entry);

            if (err != NO_ERROR) {

            status = android_log_processLogBuffer(&msg.entry, &entry);

            if (status != OK) {

                ALOGW("[%s] fails to process to an entry.\n", this->name.string());

                err = status;

                break;

            }

            lastTimestamp.tv_sec = entry.tv_sec;
@@ -688,17 +700,24 @@ status_t LogSection::BlockingCall(unique_fd& pipeWriteFd) const { 
                        trimTail(entry.message, entry.messageLen));

            proto.end(token);

        }

        if (!proto.flush(pipeWriteFd.get())) {

            if (errno == EPIPE) {

                ALOGW("[%s] wrote to a broken pipe\n", this->name.string());

            }

    gLastLogsRetrieved[mLogID] = lastTimestamp;

    if (!proto.flush(pipeWriteFd.get()) && errno == EPIPE) {

        ALOGE("[%s] wrote to a broken pipe\n", this->name.string());

        return EPIPE;

            err = errno;

            break;

        }

    return NO_ERROR;

        proto.clear();

    }

    gLastLogsRetrieved[mLogID] = lastTimestamp;

    _exit(err);

}



// ================================================================================



const int LINK_NAME_LEN = 64;

const int EXE_NAME_LEN = 1024;



TombstoneSection::TombstoneSection(int id, const char* type, const int64_t timeoutMs)

    : WorkerThreadSection(id, timeoutMs), mType(type) {

    name = "tombstone ";
@@ -716,25 +735,37 @@ status_t TombstoneSection::BlockingCall(unique_fd& pipeWriteFd) const { 


    const std::set<int> hal_pids = get_interesting_hal_pids();



    ProtoOutputStream proto;

    auto pooledBuffer = get_buffer_from_pool();

    ProtoOutputStream proto(pooledBuffer);

    // dumpBufferSize should be a multiple of page size (4 KB) to reduce memory fragmentation

    size_t dumpBufferSize = 64 * 1024; // 64 KB is enough for most tombstone dump

    char* dumpBuffer = (char*)mmap(NULL, dumpBufferSize, PROT_READ | PROT_WRITE,

                MAP_ANONYMOUS | MAP_PRIVATE, -1, 0);

    struct dirent* d;

    char link_name[LINK_NAME_LEN];

    char exe_name[EXE_NAME_LEN];

    status_t err = NO_ERROR;

    while ((d = readdir(proc.get()))) {

        int pid = atoi(d->d_name);

        if (pid <= 0) {

            continue;

        }



        const std::string link_name = android::base::StringPrintf("/proc/%d/exe", pid);

        std::string exe;

        if (!android::base::Readlink(link_name, &exe)) {

            ALOGE("Section %s: Can't read '%s': %s\n", name.string(),

                    link_name.c_str(), strerror(errno));

        snprintf(link_name, LINK_NAME_LEN, "/proc/%d/exe", pid);

        struct stat fileStat;

        if (stat(link_name, &fileStat) != OK) {

            continue;

        }

        size_t exe_name_len = readlink(link_name, exe_name, EXE_NAME_LEN);

        if (exe_name_len < 0 || exe_name_len >= EXE_NAME_LEN) {

            ALOGE("[%s] Can't read '%s': %s", name.string(), link_name, strerror(errno));

            continue;

        }

        // readlink(2) does not put a null terminator at the end

        exe_name[exe_name_len] = '\0';



        bool is_java_process;

        if (exe == "/system/bin/app_process32" || exe == "/system/bin/app_process64") {

        if (strncmp(exe_name, "/system/bin/app_process32", LINK_NAME_LEN) == 0 ||

                strncmp(exe_name, "/system/bin/app_process64", LINK_NAME_LEN) == 0) {

            if (mType != "java") continue;

            // Don't bother dumping backtraces for the zygote.

            if (IsZygote(pid)) {
@@ -743,7 +774,7 @@ status_t TombstoneSection::BlockingCall(unique_fd& pipeWriteFd) const { 
            }



            is_java_process = true;

        } else if (should_dump_native_traces(exe.c_str())) {

        } else if (should_dump_native_traces(exe_name)) {

            if (mType != "native") continue;

            is_java_process = false;

        } else if (hal_pids.find(pid) != hal_pids.end()) {
@@ -799,29 +830,37 @@ status_t TombstoneSection::BlockingCall(unique_fd& pipeWriteFd) const { 
            ALOGE("[%s] child had an issue: %s\n", this->name.string(), strerror(-cStatus));

        }



        auto dump = std::make_unique<char[]>(buffer.size());

        // Resize dump buffer

        if (dumpBufferSize < buffer.size()) {

            munmap(dumpBuffer, dumpBufferSize);

            while(dumpBufferSize < buffer.size()) dumpBufferSize = dumpBufferSize << 1;

            dumpBuffer = (char*)mmap(NULL, dumpBufferSize, PROT_READ | PROT_WRITE,

                    MAP_ANONYMOUS | MAP_PRIVATE, -1, 0);

        }

        sp<ProtoReader> reader = buffer.data()->read();

        int i = 0;

        while (reader->hasNext()) {

            dump[i] = reader->next();

            dumpBuffer[i] = reader->next();

            i++;

        }

        uint64_t token = proto.start(android::os::BackTraceProto::TRACES);

        proto.write(android::os::BackTraceProto::Stack::PID, pid);

        proto.write(android::os::BackTraceProto::Stack::DUMP, dump.get(), i);

        proto.write(android::os::BackTraceProto::Stack::DUMP, dumpBuffer, i);

        proto.write(android::os::BackTraceProto::Stack::DUMP_DURATION_NS,

                    static_cast<long long>(Nanotime() - start));

        proto.end(token);

        dumpPipe.readFd().reset();

    }



    if (!proto.flush(pipeWriteFd.get()) && errno == EPIPE) {

        if (!proto.flush(pipeWriteFd.get())) {

            if (errno == EPIPE) {

                ALOGE("[%s] wrote to a broken pipe\n", this->name.string());

        if (err != NO_ERROR) {

            return EPIPE;

            }

            err = errno;

            break;

        }



        proto.clear();

    }

    munmap(dumpBuffer, dumpBufferSize);

    return_buffer_to_pool(pooledBuffer);

    return err;

}