Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit b142cc8a authored by Yangster-mac's avatar Yangster-mac Committed by Yang Lu
Browse files

Statsd config TTL 

Roughly check the config every hour to see whether the ttl expired.
If so, read the config from disk and recreate the metric manager.

Test: statsd test

BUG: b/77274363

Change-Id: I16838afe5bbe966c3a0f638869751f9b59a5a259
parent a926126a

cmds/statsd/Android.mk

+2 −1
Original line number Diff line number Diff line
@@ -209,7 +209,8 @@ LOCAL_SRC_FILES := \ 
    tests/e2e/DimensionInCondition_e2e_combination_OR_cond_test.cpp \

    tests/e2e/DimensionInCondition_e2e_simple_cond_test.cpp \

    tests/e2e/Anomaly_count_e2e_test.cpp \

    tests/e2e/Anomaly_duration_sum_e2e_test.cpp

    tests/e2e/Anomaly_duration_sum_e2e_test.cpp \

    tests/e2e/ConfigTtl_e2e_test.cpp



LOCAL_STATIC_LIBRARIES := \

    $(statsd_common_static_libraries) \

cmds/statsd/benchmark/metric_util.cpp

+1 −1
Original line number Diff line number Diff line
@@ -366,7 +366,7 @@ sp<StatsLogProcessor> CreateStatsLogProcessor(const long timeBaseSec, const Stat 
    sp<AlarmMonitor> periodicAlarmMonitor;

    sp<StatsLogProcessor> processor = new StatsLogProcessor(

        uidMap, anomalyAlarmMonitor, periodicAlarmMonitor, timeBaseSec, [](const ConfigKey&){});

    processor->OnConfigUpdated(0, key, config);

    processor->OnConfigUpdated(timeBaseSec * NS_PER_SEC, key, config);

    return processor;

}

cmds/statsd/src/StatsLogProcessor.cpp

+42 −8
Original line number Diff line number Diff line
@@ -65,6 +65,7 @@ const int FIELD_ID_CURRENT_REPORT_ELAPSED_NANOS = 4; 
const int FIELD_ID_LAST_REPORT_WALL_CLOCK_NANOS = 5;

const int FIELD_ID_CURRENT_REPORT_WALL_CLOCK_NANOS = 6;



#define NS_PER_HOUR 3600 * NS_PER_SEC



#define STATS_DATA_DIR "/data/misc/stats-data"
@@ -85,7 +86,7 @@ StatsLogProcessor::~StatsLogProcessor() { 
}



void StatsLogProcessor::onAnomalyAlarmFired(

        const uint64_t& timestampNs,

        const int64_t& timestampNs,

        unordered_set<sp<const InternalAlarm>, SpHash<InternalAlarm>> alarmSet) {

    std::lock_guard<std::mutex> lock(mMetricsMutex);

    for (const auto& itr : mMetricsManagers) {
@@ -93,7 +94,7 @@ void StatsLogProcessor::onAnomalyAlarmFired( 
    }

}

void StatsLogProcessor::onPeriodicAlarmFired(

        const uint64_t& timestampNs,

        const int64_t& timestampNs,

        unordered_set<sp<const InternalAlarm>, SpHash<InternalAlarm>> alarmSet) {



    std::lock_guard<std::mutex> lock(mMetricsMutex);
@@ -156,10 +157,14 @@ void StatsLogProcessor::onIsolatedUidChangedEventLocked(const LogEvent& event) { 


void StatsLogProcessor::OnLogEvent(LogEvent* event) {

    std::lock_guard<std::mutex> lock(mMetricsMutex);

    if (event->GetElapsedTimestampNs() < mLastLogTimestamp) {

    const int64_t currentTimestampNs = event->GetElapsedTimestampNs();

    if (currentTimestampNs < mLastLogTimestamp) {

        return;

    }

    mLastLogTimestamp = event->GetElapsedTimestampNs();



    resetIfConfigTtlExpiredLocked(currentTimestampNs);



    mLastLogTimestamp = currentTimestampNs;

    StatsdStats::getInstance().noteAtomLogged(

        event->GetTagId(), event->GetElapsedTimestampNs() / NS_PER_SEC);
@@ -173,12 +178,13 @@ void StatsLogProcessor::OnLogEvent(LogEvent* event) { 
        return;

    }



    uint64_t curTimeSec = getElapsedRealtimeSec();

    int64_t curTimeSec = getElapsedRealtimeSec();

    if (curTimeSec - mLastPullerCacheClearTimeSec > StatsdStats::kPullerCacheClearIntervalSec) {

        mStatsPullerManager.ClearPullerCacheIfNecessary(curTimeSec * NS_PER_SEC);

        mLastPullerCacheClearTimeSec = curTimeSec;

    }





    if (event->GetTagId() != android::util::ISOLATED_UID_CHANGED) {

        // Map the isolated uid to host uid if necessary.

        mapIsolatedUidToHostUidIfNecessaryLocked(event);
@@ -194,6 +200,11 @@ void StatsLogProcessor::OnLogEvent(LogEvent* event) { 
void StatsLogProcessor::OnConfigUpdated(const int64_t timestampNs, const ConfigKey& key,

                                        const StatsdConfig& config) {

    std::lock_guard<std::mutex> lock(mMetricsMutex);

    OnConfigUpdatedLocked(timestampNs, key, config);

}



void StatsLogProcessor::OnConfigUpdatedLocked(

        const int64_t timestampNs, const ConfigKey& key, const StatsdConfig& config) {

    VLOG("Updated configuration for key %s", key.ToString().c_str());

    sp<MetricsManager> newMetricsManager =

        new MetricsManager(key, config, mTimeBaseSec, (timestampNs - 1) / NS_PER_SEC + 1, mUidMap,
@@ -206,6 +217,7 @@ void StatsLogProcessor::OnConfigUpdated(const int64_t timestampNs, const ConfigK 
            // not safe to create wp or sp from this pointer inside its constructor.

            mUidMap->addListener(newMetricsManager.get());

        }

        newMetricsManager->refreshTtl(timestampNs);

        mMetricsManagers[key] = newMetricsManager;

        VLOG("StatsdConfig valid");

    } else {
@@ -235,7 +247,7 @@ void StatsLogProcessor::dumpStates(FILE* out, bool verbose) { 
/*

 * onDumpReport dumps serialized ConfigMetricsReportList into outData.

 */

void StatsLogProcessor::onDumpReport(const ConfigKey& key, const uint64_t dumpTimeStampNs,

void StatsLogProcessor::onDumpReport(const ConfigKey& key, const int64_t dumpTimeStampNs,

                                     vector<uint8_t>* outData) {

    std::lock_guard<std::mutex> lock(mMetricsMutex);
@@ -290,7 +302,7 @@ void StatsLogProcessor::onDumpReport(const ConfigKey& key, const uint64_t dumpTi 
 * onConfigMetricsReportLocked dumps serialized ConfigMetricsReport into outData.

 */

void StatsLogProcessor::onConfigMetricsReportLocked(const ConfigKey& key,

                                                    const uint64_t dumpTimeStampNs,

                                                    const int64_t dumpTimeStampNs,

                                                    ProtoOutputStream* proto) {

    // We already checked whether key exists in mMetricsManagers in

    // WriteDataToDisk.
@@ -317,7 +329,29 @@ void StatsLogProcessor::onConfigMetricsReportLocked(const ConfigKey& key, 
    proto->write(FIELD_TYPE_INT64 | FIELD_ID_CURRENT_REPORT_WALL_CLOCK_NANOS,

                (long long)getWallClockNs());



}



void StatsLogProcessor::resetIfConfigTtlExpiredLocked(const int64_t timestampNs) {

    std::vector<ConfigKey> configKeysTtlExpired;

    for (auto it = mMetricsManagers.begin(); it != mMetricsManagers.end(); it++) {

        if (it->second != nullptr && !it->second->isInTtl(timestampNs)) {

            configKeysTtlExpired.push_back(it->first);

        }

    }



    for (const auto& key : configKeysTtlExpired) {

        StatsdConfig config;

        if (StorageManager::readConfigFromDisk(key, &config)) {

            OnConfigUpdatedLocked(timestampNs, key, config);

            StatsdStats::getInstance().noteConfigReset(key);

        } else {

            ALOGE("Failed to read backup config from disk for : %s", key.ToString().c_str());

            auto it = mMetricsManagers.find(key);

            if (it != mMetricsManagers.end()) {

                it->second->refreshTtl(timestampNs);

            }

        }

    }

}



void StatsLogProcessor::OnConfigRemoved(const ConfigKey& key) {
@@ -337,7 +371,7 @@ void StatsLogProcessor::OnConfigRemoved(const ConfigKey& key) { 
}



void StatsLogProcessor::flushIfNecessaryLocked(

    uint64_t timestampNs, const ConfigKey& key, MetricsManager& metricsManager) {

    int64_t timestampNs, const ConfigKey& key, MetricsManager& metricsManager) {

    auto lastCheckTime = mLastByteSizeTimes.find(key);

    if (lastCheckTime != mLastByteSizeTimes.end()) {

        if (timestampNs - lastCheckTime->second < StatsdStats::kMinByteSizeCheckPeriodNs) {

cmds/statsd/src/StatsLogProcessor.h

+13 −5
Original line number Diff line number Diff line
@@ -48,16 +48,16 @@ public: 


    size_t GetMetricsSize(const ConfigKey& key) const;



    void onDumpReport(const ConfigKey& key, const uint64_t dumpTimeNs, vector<uint8_t>* outData);

    void onDumpReport(const ConfigKey& key, const int64_t dumpTimeNs, vector<uint8_t>* outData);



    /* Tells MetricsManager that the alarms in alarmSet have fired. Modifies anomaly alarmSet. */

    void onAnomalyAlarmFired(

            const uint64_t& timestampNs,

            const int64_t& timestampNs,

            unordered_set<sp<const InternalAlarm>, SpHash<InternalAlarm>> alarmSet);



    /* Tells MetricsManager that the alarms in alarmSet have fired. Modifies periodic alarmSet. */

    void onPeriodicAlarmFired(

            const uint64_t& timestampNs,

            const int64_t& timestampNs,

            unordered_set<sp<const InternalAlarm>, SpHash<InternalAlarm>> alarmSet);



    /* Flushes data to disk. Data on memory will be gone after written to disk. */
@@ -69,6 +69,7 @@ public: 


    void dumpStates(FILE* out, bool verbose);





private:

    // For testing only.

    inline sp<AlarmMonitor> getAnomalyAlarmMonitor() const {
@@ -96,12 +97,17 @@ private: 


    sp<AlarmMonitor> mPeriodicAlarmMonitor;



    void onConfigMetricsReportLocked(const ConfigKey& key, const uint64_t dumpTimeStampNs,

    void resetIfConfigTtlExpiredLocked(const int64_t timestampNs);



    void OnConfigUpdatedLocked(

        const int64_t currentTimestampNs, const ConfigKey& key, const StatsdConfig& config);



    void onConfigMetricsReportLocked(const ConfigKey& key, const int64_t dumpTimeStampNs,

                                     util::ProtoOutputStream* proto);



    /* Check if we should send a broadcast if approaching memory limits and if we're over, we

     * actually delete the data. */

    void flushIfNecessaryLocked(uint64_t timestampNs, const ConfigKey& key,

    void flushIfNecessaryLocked(int64_t timestampNs, const ConfigKey& key,

                                MetricsManager& metricsManager);



    // Maps the isolated uid in the log event to host uid if the log event contains uid fields.
@@ -152,7 +158,9 @@ private: 
    FRIEND_TEST(AnomalyDetectionE2eTest, TestDurationMetric_SUM_single_bucket);

    FRIEND_TEST(AnomalyDetectionE2eTest, TestDurationMetric_SUM_multiple_buckets);

    FRIEND_TEST(AnomalyDetectionE2eTest, TestDurationMetric_SUM_long_refractory_period);



    FRIEND_TEST(AlarmE2eTest, TestMultipleAlarms);

    FRIEND_TEST(ConfigTtlE2eTest, TestCountMetric);

};



}  // namespace statsd

cmds/statsd/src/StatsService.cpp

+2 −2
Original line number Diff line number Diff line
@@ -677,7 +677,7 @@ Status StatsService::informAnomalyAlarmFired() { 
                                         "Only system uid can call informAnomalyAlarmFired");

    }



    uint64_t currentTimeSec = getElapsedRealtimeSec();

    int64_t currentTimeSec = getElapsedRealtimeSec();

    std::unordered_set<sp<const InternalAlarm>, SpHash<InternalAlarm>> alarmSet =

            mAnomalyAlarmMonitor->popSoonerThan(static_cast<uint32_t>(currentTimeSec));

    if (alarmSet.size() > 0) {
@@ -698,7 +698,7 @@ Status StatsService::informAlarmForSubscriberTriggeringFired() { 
                "Only system uid can call informAlarmForSubscriberTriggeringFired");

    }



    uint64_t currentTimeSec = getElapsedRealtimeSec();

    int64_t currentTimeSec = getElapsedRealtimeSec();

    std::unordered_set<sp<const InternalAlarm>, SpHash<InternalAlarm>> alarmSet =

            mPeriodicAlarmMonitor->popSoonerThan(static_cast<uint32_t>(currentTimeSec));

    if (alarmSet.size() > 0) {