Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit e68f3a58 authored by Yangster-mac's avatar Yangster-mac
Browse files

Flush the partial bucket when startd shuts down or config updated. 

Test: statsd test

BUG: b/77556036
Change-Id: Ie4a04ace55e07c4529cdff5906ba874f8815f620
parent 203bbbf9

cmds/statsd/src/StatsLogProcessor.cpp

+50 −36
Original line number Original line Diff line number Diff line
@@ -212,7 +212,10 @@ void StatsLogProcessor::OnConfigUpdatedLocked( 
    sp<MetricsManager> newMetricsManager =

    sp<MetricsManager> newMetricsManager =

        new MetricsManager(key, config, mTimeBaseSec, (timestampNs - 1) / NS_PER_SEC + 1, mUidMap,

        new MetricsManager(key, config, mTimeBaseSec, (timestampNs - 1) / NS_PER_SEC + 1, mUidMap,

                           mAnomalyAlarmMonitor, mPeriodicAlarmMonitor);

                           mAnomalyAlarmMonitor, mPeriodicAlarmMonitor);



    auto it = mMetricsManagers.find(key);

    if (it != mMetricsManagers.end()) {

        WriteDataToDiskLocked(it->first);

    }

    if (newMetricsManager->isConfigValid()) {

    if (newMetricsManager->isConfigValid()) {

        mUidMap->OnConfigUpdated(key);

        mUidMap->OnConfigUpdated(key);

        if (newMetricsManager->shouldAddUidMapListener()) {

        if (newMetricsManager->shouldAddUidMapListener()) {
@@ -251,19 +254,10 @@ void StatsLogProcessor::dumpStates(FILE* out, bool verbose) { 
 * onDumpReport dumps serialized ConfigMetricsReportList into outData.

 * onDumpReport dumps serialized ConfigMetricsReportList into outData.

 */

 */

void StatsLogProcessor::onDumpReport(const ConfigKey& key, const int64_t dumpTimeStampNs,

void StatsLogProcessor::onDumpReport(const ConfigKey& key, const int64_t dumpTimeStampNs,

                                     const bool include_current_partial_bucket,

                                     vector<uint8_t>* outData) {

                                     vector<uint8_t>* outData) {

    std::lock_guard<std::mutex> lock(mMetricsMutex);

    std::lock_guard<std::mutex> lock(mMetricsMutex);





    auto it = mMetricsManagers.find(key);

    if (it == mMetricsManagers.end()) {

        ALOGW("Config source %s does not exist", key.ToString().c_str());

        return;

    }



    // This allows another broadcast to be sent within the rate-limit period if we get close to

    // filling the buffer again soon.

    mLastBroadcastTimes.erase(key);



    ProtoOutputStream proto;

    ProtoOutputStream proto;





    // Start of ConfigKey.

    // Start of ConfigKey.
@@ -273,17 +267,25 @@ void StatsLogProcessor::onDumpReport(const ConfigKey& key, const int64_t dumpTim 
    proto.end(configKeyToken);

    proto.end(configKeyToken);

    // End of ConfigKey.

    // End of ConfigKey.





    // Then, check stats-data directory to see there's any file containing

    // ConfigMetricsReport from previous shutdowns to concatenate to reports.

    StorageManager::appendConfigMetricsReport(key, &proto);



    auto it = mMetricsManagers.find(key);

    if (it != mMetricsManagers.end()) {

        // This allows another broadcast to be sent within the rate-limit period if we get close to

        // filling the buffer again soon.

        mLastBroadcastTimes.erase(key);



        // Start of ConfigMetricsReport (reports).

        // Start of ConfigMetricsReport (reports).

        uint64_t reportsToken =

        uint64_t reportsToken =

                proto.start(FIELD_TYPE_MESSAGE | FIELD_COUNT_REPEATED | FIELD_ID_REPORTS);

                proto.start(FIELD_TYPE_MESSAGE | FIELD_COUNT_REPEATED | FIELD_ID_REPORTS);

    onConfigMetricsReportLocked(key, dumpTimeStampNs, &proto);

        onConfigMetricsReportLocked(key, dumpTimeStampNs, include_current_partial_bucket, &proto);

        proto.end(reportsToken);

        proto.end(reportsToken);

        // End of ConfigMetricsReport (reports).

        // End of ConfigMetricsReport (reports).



    } else {



        ALOGW("Config source %s does not exist", key.ToString().c_str());

    // Then, check stats-data directory to see there's any file containing

    }

    // ConfigMetricsReport from previous shutdowns to concatenate to reports.

    StorageManager::appendConfigMetricsReport(key, &proto);





    if (outData != nullptr) {

    if (outData != nullptr) {

        outData->clear();

        outData->clear();
@@ -298,7 +300,7 @@ void StatsLogProcessor::onDumpReport(const ConfigKey& key, const int64_t dumpTim 
        }

        }

    }

    }





    StatsdStats::getInstance().noteMetricsReportSent(key);

    StatsdStats::getInstance().noteMetricsReportSent(key, proto.size());

}

}





/*

/*
@@ -306,16 +308,20 @@ void StatsLogProcessor::onDumpReport(const ConfigKey& key, const int64_t dumpTim 
 */

 */

void StatsLogProcessor::onConfigMetricsReportLocked(const ConfigKey& key,

void StatsLogProcessor::onConfigMetricsReportLocked(const ConfigKey& key,

                                                    const int64_t dumpTimeStampNs,

                                                    const int64_t dumpTimeStampNs,

                                                    const bool include_current_partial_bucket,

                                                    ProtoOutputStream* proto) {

                                                    ProtoOutputStream* proto) {

    // We already checked whether key exists in mMetricsManagers in

    // We already checked whether key exists in mMetricsManagers in

    // WriteDataToDisk.

    // WriteDataToDisk.

    auto it = mMetricsManagers.find(key);

    auto it = mMetricsManagers.find(key);

    if (it == mMetricsManagers.end()) {

        return;

    }

    int64_t lastReportTimeNs = it->second->getLastReportTimeNs();

    int64_t lastReportTimeNs = it->second->getLastReportTimeNs();

    int64_t lastReportWallClockNs = it->second->getLastReportWallClockNs();

    int64_t lastReportWallClockNs = it->second->getLastReportWallClockNs();





    // First, fill in ConfigMetricsReport using current data on memory, which

    // First, fill in ConfigMetricsReport using current data on memory, which

    // starts from filling in StatsLogReport's.

    // starts from filling in StatsLogReport's.

    it->second->onDumpReport(dumpTimeStampNs, proto);

    it->second->onDumpReport(dumpTimeStampNs, include_current_partial_bucket, proto);





    // Fill in UidMap.

    // Fill in UidMap.

    uint64_t uidMapToken = proto->start(FIELD_TYPE_MESSAGE | FIELD_ID_UID_MAP);

    uint64_t uidMapToken = proto->start(FIELD_TYPE_MESSAGE | FIELD_ID_UID_MAP);
@@ -331,7 +337,6 @@ void StatsLogProcessor::onConfigMetricsReportLocked(const ConfigKey& key, 
                (long long)lastReportWallClockNs);

                (long long)lastReportWallClockNs);

    proto->write(FIELD_TYPE_INT64 | FIELD_ID_CURRENT_REPORT_WALL_CLOCK_NANOS,

    proto->write(FIELD_TYPE_INT64 | FIELD_ID_CURRENT_REPORT_WALL_CLOCK_NANOS,

                (long long)getWallClockNs());

                (long long)getWallClockNs());



}

}





void StatsLogProcessor::resetIfConfigTtlExpiredLocked(const int64_t timestampNs) {

void StatsLogProcessor::resetIfConfigTtlExpiredLocked(const int64_t timestampNs) {
@@ -361,6 +366,7 @@ void StatsLogProcessor::OnConfigRemoved(const ConfigKey& key) { 
    std::lock_guard<std::mutex> lock(mMetricsMutex);

    std::lock_guard<std::mutex> lock(mMetricsMutex);

    auto it = mMetricsManagers.find(key);

    auto it = mMetricsManagers.find(key);

    if (it != mMetricsManagers.end()) {

    if (it != mMetricsManagers.end()) {

        WriteDataToDiskLocked(key);

        mMetricsManagers.erase(it);

        mMetricsManagers.erase(it);

        mUidMap->OnConfigRemoved(key);

        mUidMap->OnConfigRemoved(key);

    }

    }
@@ -406,22 +412,30 @@ void StatsLogProcessor::flushIfNecessaryLocked( 
    }

    }

}

}





void StatsLogProcessor::WriteDataToDisk() {

void StatsLogProcessor::WriteDataToDiskLocked(const ConfigKey& key) {

    std::lock_guard<std::mutex> lock(mMetricsMutex);

    for (auto& pair : mMetricsManagers) {

        const ConfigKey& key = pair.first;

    ProtoOutputStream proto;

    ProtoOutputStream proto;

        onConfigMetricsReportLocked(key, getElapsedRealtimeNs(), &proto);

    onConfigMetricsReportLocked(key, getElapsedRealtimeNs(),

                                true /* include_current_partial_bucket*/, &proto);

    string file_name = StringPrintf("%s/%ld_%d_%lld", STATS_DATA_DIR,

    string file_name = StringPrintf("%s/%ld_%d_%lld", STATS_DATA_DIR,

         (long)getWallClockSec(), key.GetUid(), (long long)key.GetId());

         (long)getWallClockSec(), key.GetUid(), (long long)key.GetId());

    android::base::unique_fd fd(open(file_name.c_str(),

    android::base::unique_fd fd(open(file_name.c_str(),

                                O_WRONLY | O_CREAT | O_CLOEXEC, S_IRUSR | S_IWUSR));

                                O_WRONLY | O_CREAT | O_CLOEXEC, S_IRUSR | S_IWUSR));

    if (fd == -1) {

    if (fd == -1) {

            VLOG("Attempt to write %s but failed", file_name.c_str());

        ALOGE("Attempt to write %s but failed", file_name.c_str());

        return;

        return;

    }

    }

    proto.flush(fd.get());

    proto.flush(fd.get());

}

}



void StatsLogProcessor::WriteDataToDiskLocked() {

    for (auto& pair : mMetricsManagers) {

        WriteDataToDiskLocked(pair.first);

    }

}



void StatsLogProcessor::WriteDataToDisk() {

    std::lock_guard<std::mutex> lock(mMetricsMutex);

    WriteDataToDiskLocked();

}

}





}  // namespace statsd

}  // namespace statsd

cmds/statsd/src/StatsLogProcessor.h

+6 −1
Original line number Original line Diff line number Diff line
@@ -48,7 +48,8 @@ public: 




    size_t GetMetricsSize(const ConfigKey& key) const;

    size_t GetMetricsSize(const ConfigKey& key) const;





    void onDumpReport(const ConfigKey& key, const int64_t dumpTimeNs, vector<uint8_t>* outData);

    void onDumpReport(const ConfigKey& key, const int64_t dumpTimeNs,

                      const bool include_current_partial_bucket, vector<uint8_t>* outData);





    /* Tells MetricsManager that the alarms in alarmSet have fired. Modifies anomaly alarmSet. */

    /* Tells MetricsManager that the alarms in alarmSet have fired. Modifies anomaly alarmSet. */

    void onAnomalyAlarmFired(

    void onAnomalyAlarmFired(
@@ -102,7 +103,11 @@ private: 
    void OnConfigUpdatedLocked(

    void OnConfigUpdatedLocked(

        const int64_t currentTimestampNs, const ConfigKey& key, const StatsdConfig& config);

        const int64_t currentTimestampNs, const ConfigKey& key, const StatsdConfig& config);





    void WriteDataToDiskLocked();

    void WriteDataToDiskLocked(const ConfigKey& key);



    void onConfigMetricsReportLocked(const ConfigKey& key, const int64_t dumpTimeStampNs,

    void onConfigMetricsReportLocked(const ConfigKey& key, const int64_t dumpTimeStampNs,

                                     const bool include_current_partial_bucket,

                                     util::ProtoOutputStream* proto);

                                     util::ProtoOutputStream* proto);





    /* Check if we should send a broadcast if approaching memory limits and if we're over, we

    /* Check if we should send a broadcast if approaching memory limits and if we're over, we

cmds/statsd/src/StatsService.cpp

+3 −2
Original line number Original line Diff line number Diff line
@@ -501,7 +501,7 @@ status_t StatsService::cmd_dump_report(FILE* out, FILE* err, const Vector<String 
        if (good) {

        if (good) {

            vector<uint8_t> data;

            vector<uint8_t> data;

            mProcessor->onDumpReport(ConfigKey(uid, StrToInt64(name)), getElapsedRealtimeNs(),

            mProcessor->onDumpReport(ConfigKey(uid, StrToInt64(name)), getElapsedRealtimeNs(),

                                     &data);

                                     false /* include_current_bucket*/, &data);

            // TODO: print the returned StatsLogReport to file instead of printing to logcat.

            // TODO: print the returned StatsLogReport to file instead of printing to logcat.

            if (proto) {

            if (proto) {

                for (size_t i = 0; i < data.size(); i ++) {

                for (size_t i = 0; i < data.size(); i ++) {
@@ -800,7 +800,8 @@ Status StatsService::getData(int64_t key, vector<uint8_t>* output) { 
    VLOG("StatsService::getData with Pid %i, Uid %i", ipc->getCallingPid(), ipc->getCallingUid());

    VLOG("StatsService::getData with Pid %i, Uid %i", ipc->getCallingPid(), ipc->getCallingUid());

    if (checkCallingPermission(String16(kPermissionDump))) {

    if (checkCallingPermission(String16(kPermissionDump))) {

        ConfigKey configKey(ipc->getCallingUid(), key);

        ConfigKey configKey(ipc->getCallingUid(), key);

        mProcessor->onDumpReport(configKey, getElapsedRealtimeNs(), output);

        mProcessor->onDumpReport(configKey, getElapsedRealtimeNs(),

                                 false /* include_current_bucket*/, output);

        return Status::ok();

        return Status::ok();

    } else {

    } else {

        return Status::fromExceptionCode(binder::Status::EX_SECURITY);

        return Status::fromExceptionCode(binder::Status::EX_SECURITY);

cmds/statsd/src/guardrail/StatsdStats.cpp

+22 −13
Original line number Original line Diff line number Diff line
@@ -76,7 +76,8 @@ const int FIELD_ID_CONFIG_STATS_ALERT_COUNT = 8; 
const int FIELD_ID_CONFIG_STATS_VALID = 9;

const int FIELD_ID_CONFIG_STATS_VALID = 9;

const int FIELD_ID_CONFIG_STATS_BROADCAST = 10;

const int FIELD_ID_CONFIG_STATS_BROADCAST = 10;

const int FIELD_ID_CONFIG_STATS_DATA_DROP = 11;

const int FIELD_ID_CONFIG_STATS_DATA_DROP = 11;

const int FIELD_ID_CONFIG_STATS_DUMP_REPORT = 12;

const int FIELD_ID_CONFIG_STATS_DUMP_REPORT_TIME = 12;

const int FIELD_ID_CONFIG_STATS_DUMP_REPORT_BYTES = 20;

const int FIELD_ID_CONFIG_STATS_MATCHER_STATS = 13;

const int FIELD_ID_CONFIG_STATS_MATCHER_STATS = 13;

const int FIELD_ID_CONFIG_STATS_CONDITION_STATS = 14;

const int FIELD_ID_CONFIG_STATS_CONDITION_STATS = 14;

const int FIELD_ID_CONFIG_STATS_METRIC_STATS = 15;

const int FIELD_ID_CONFIG_STATS_METRIC_STATS = 15;
@@ -215,21 +216,22 @@ void StatsdStats::noteDataDropped(const ConfigKey& key, int32_t timeSec) { 
    it->second->data_drop_time_sec.push_back(timeSec);

    it->second->data_drop_time_sec.push_back(timeSec);

}

}





void StatsdStats::noteMetricsReportSent(const ConfigKey& key) {

void StatsdStats::noteMetricsReportSent(const ConfigKey& key, const size_t num_bytes) {

    noteMetricsReportSent(key, getWallClockSec());

    noteMetricsReportSent(key, num_bytes, getWallClockSec());

}

}





void StatsdStats::noteMetricsReportSent(const ConfigKey& key, int32_t timeSec) {

void StatsdStats::noteMetricsReportSent(const ConfigKey& key, const size_t num_bytes,

                                        int32_t timeSec) {

    lock_guard<std::mutex> lock(mLock);

    lock_guard<std::mutex> lock(mLock);

    auto it = mConfigStats.find(key);

    auto it = mConfigStats.find(key);

    if (it == mConfigStats.end()) {

    if (it == mConfigStats.end()) {

        ALOGE("Config key %s not found!", key.ToString().c_str());

        ALOGE("Config key %s not found!", key.ToString().c_str());

        return;

        return;

    }

    }

    if (it->second->dump_report_time_sec.size() == kMaxTimestampCount) {

    if (it->second->dump_report_stats.size() == kMaxTimestampCount) {

        it->second->dump_report_time_sec.pop_front();

        it->second->dump_report_stats.pop_front();

    }

    }

    it->second->dump_report_time_sec.push_back(timeSec);

    it->second->dump_report_stats.push_back(std::make_pair(timeSec, num_bytes));

}

}





void StatsdStats::noteUidMapDropped(int deltas) {

void StatsdStats::noteUidMapDropped(int deltas) {
@@ -383,7 +385,7 @@ void StatsdStats::resetInternalLocked() { 
    for (auto& config : mConfigStats) {

    for (auto& config : mConfigStats) {

        config.second->broadcast_sent_time_sec.clear();

        config.second->broadcast_sent_time_sec.clear();

        config.second->data_drop_time_sec.clear();

        config.second->data_drop_time_sec.clear();

        config.second->dump_report_time_sec.clear();

        config.second->dump_report_stats.clear();

        config.second->annotations.clear();

        config.second->annotations.clear();

        config.second->matcher_stats.clear();

        config.second->matcher_stats.clear();

        config.second->condition_stats.clear();

        config.second->condition_stats.clear();
@@ -442,8 +444,8 @@ void StatsdStats::dumpStats(FILE* out) const { 
            fprintf(out, "\tdata drop time: %d\n", dataDropTime);

            fprintf(out, "\tdata drop time: %d\n", dataDropTime);

        }

        }





        for (const auto& dumpTime : configStats->dump_report_time_sec) {

        for (const auto& dump : configStats->dump_report_stats) {

            fprintf(out, "\tdump report time: %d\n", dumpTime);

            fprintf(out, "\tdump report time: %d bytes: %lld\n", dump.first, (long long)dump.second);

        }

        }





        for (const auto& stats : pair.second->matcher_stats) {

        for (const auto& stats : pair.second->matcher_stats) {
@@ -536,9 +538,16 @@ void addConfigStatsToProto(const ConfigStats& configStats, ProtoOutputStream* pr 
                     drop);

                     drop);

    }

    }





    for (const auto& dump : configStats.dump_report_time_sec) {

    for (const auto& dump : configStats.dump_report_stats) {

        proto->write(FIELD_TYPE_INT32 | FIELD_ID_CONFIG_STATS_DUMP_REPORT | FIELD_COUNT_REPEATED,

        proto->write(FIELD_TYPE_INT32 | FIELD_ID_CONFIG_STATS_DUMP_REPORT_TIME |

                     dump);

                     FIELD_COUNT_REPEATED,

                     dump.first);

    }



    for (const auto& dump : configStats.dump_report_stats) {

        proto->write(FIELD_TYPE_INT64 | FIELD_ID_CONFIG_STATS_DUMP_REPORT_BYTES |

                     FIELD_COUNT_REPEATED,

                     (long long)dump.second);

    }

    }





    for (const auto& annotation : configStats.annotations) {

    for (const auto& annotation : configStats.annotations) {

cmds/statsd/src/guardrail/StatsdStats.h

+3 −3
Original line number Original line Diff line number Diff line
@@ -43,7 +43,7 @@ struct ConfigStats { 




    std::list<int32_t> broadcast_sent_time_sec;

    std::list<int32_t> broadcast_sent_time_sec;

    std::list<int32_t> data_drop_time_sec;

    std::list<int32_t> data_drop_time_sec;

    std::list<int32_t> dump_report_time_sec;

    std::list<std::pair<int32_t, int64_t>> dump_report_stats;





    // Stores how many times a matcher have been matched. The map size is capped by kMaxConfigCount.

    // Stores how many times a matcher have been matched. The map size is capped by kMaxConfigCount.

    std::map<const int64_t, int> matcher_stats;

    std::map<const int64_t, int> matcher_stats;
@@ -177,7 +177,7 @@ public: 
     *

     *

     * The report may be requested via StatsManager API, or through adb cmd.

     * The report may be requested via StatsManager API, or through adb cmd.

     */

     */

    void noteMetricsReportSent(const ConfigKey& key);

    void noteMetricsReportSent(const ConfigKey& key, const size_t num_bytes);





    /**

    /**

     * Report the size of output tuple of a condition.

     * Report the size of output tuple of a condition.
@@ -355,7 +355,7 @@ private: 




    void noteDataDropped(const ConfigKey& key, int32_t timeSec);

    void noteDataDropped(const ConfigKey& key, int32_t timeSec);





    void noteMetricsReportSent(const ConfigKey& key, int32_t timeSec);

    void noteMetricsReportSent(const ConfigKey& key, const size_t num_bytes, int32_t timeSec);





    void noteBroadcastSent(const ConfigKey& key, int32_t timeSec);

    void noteBroadcastSent(const ConfigKey& key, int32_t timeSec);