Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit f09569f8 authored by Yao Chen's avatar Yao Chen
Browse files

Further reduce statsd memory usage. 

+ Remove the protobuf *Metric object from MetricProducers
   -- This saves ~150 bytes per metric.
+ Remove the StatsdConfig from ConfigManager
   -- This saves us xKB per config.
+ Also remove alerts from fake config to avoid crash (Bug: 70627390)
+ Other misc fixes too.

Test: statsd_test & manual
Change-Id: Ied4eb3fa31c50599817b3a5e1caf5077c487fad2
parent 14b1f3ed

cmds/statsd/src/config/ConfigManager.cpp

+36 −22
Original line number Diff line number Diff line
@@ -29,6 +29,12 @@ namespace android { 
namespace os {

namespace statsd {



using std::map;

using std::pair;

using std::set;

using std::string;

using std::vector;



#define STATS_SERVICE_DIR "/data/misc/stats-service"



using android::base::StringPrintf;
@@ -41,8 +47,14 @@ ConfigManager::~ConfigManager() { 
}



void ConfigManager::Startup() {

    StorageManager::readConfigFromDisk(mConfigs);



    map<ConfigKey, StatsdConfig> configsFromDisk;

    StorageManager::readConfigFromDisk(configsFromDisk);

    // TODO(b/70667694): Make the configs from disk be used. And remove the fake config,

    // and tests shouldn't call this Startup(), maybe call StartupForTest() so we don't read

    // configs from disk for tests.

    // for (const auto& pair : configsFromDisk) {

    //    UpdateConfig(pair.first, pair.second);

    //}

    // this should be called from StatsService when it receives a statsd_config

    UpdateConfig(ConfigKey(1000, "fake"), build_fake_config());

}
@@ -52,9 +64,8 @@ void ConfigManager::AddListener(const sp<ConfigListener>& listener) { 
}



void ConfigManager::UpdateConfig(const ConfigKey& key, const StatsdConfig& config) {

    // Add to map

    mConfigs[key] = config;

    // Why doesn't this work? mConfigs.insert({key, config});

    // Add to set

    mConfigs.insert(key);



    // Save to disk

    update_saved_configs(key, config);
@@ -74,7 +85,7 @@ void ConfigManager::RemoveConfigReceiver(const ConfigKey& key) { 
}



void ConfigManager::RemoveConfig(const ConfigKey& key) {

    unordered_map<ConfigKey, StatsdConfig>::iterator it = mConfigs.find(key);

    auto it = mConfigs.find(key);

    if (it != mConfigs.end()) {

        // Remove from map

        mConfigs.erase(it);
@@ -100,9 +111,9 @@ void ConfigManager::RemoveConfigs(int uid) { 


    for (auto it = mConfigs.begin(); it != mConfigs.end();) {

        // Remove from map

        if (it->first.GetUid() == uid) {

            removed.push_back(it->first);

            mConfigReceivers.erase(it->first);

        if (it->GetUid() == uid) {

            removed.push_back(*it);

            mConfigReceivers.erase(*it);

            it = mConfigs.erase(it);

        } else {

            it++;
@@ -123,10 +134,10 @@ void ConfigManager::RemoveAllConfigs() { 


    for (auto it = mConfigs.begin(); it != mConfigs.end();) {

        // Remove from map

        removed.push_back(it->first);

        auto receiverIt = mConfigReceivers.find(it->first);

        removed.push_back(*it);

        auto receiverIt = mConfigReceivers.find(*it);

        if (receiverIt != mConfigReceivers.end()) {

            mConfigReceivers.erase(it->first);

            mConfigReceivers.erase(*it);

        }

        it = mConfigs.erase(it);

    }
@@ -143,7 +154,7 @@ void ConfigManager::RemoveAllConfigs() { 
vector<ConfigKey> ConfigManager::GetAllConfigKeys() const {

    vector<ConfigKey> ret;

    for (auto it = mConfigs.cbegin(); it != mConfigs.cend(); ++it) {

        ret.push_back(it->first);

        ret.push_back(*it);

    }

    return ret;

}
@@ -160,15 +171,13 @@ const pair<string, string> ConfigManager::GetConfigReceiver(const ConfigKey& key 
void ConfigManager::Dump(FILE* out) {

    fprintf(out, "CONFIGURATIONS (%d)\n", (int)mConfigs.size());

    fprintf(out, "     uid name\n");

    for (unordered_map<ConfigKey, StatsdConfig>::const_iterator it = mConfigs.begin();

         it != mConfigs.end(); it++) {

        fprintf(out, "  %6d %s\n", it->first.GetUid(), it->first.GetName().c_str());

        auto receiverIt = mConfigReceivers.find(it->first);

    for (const auto& key : mConfigs) {

        fprintf(out, "  %6d %s\n", key.GetUid(), key.GetName().c_str());

        auto receiverIt = mConfigReceivers.find(key);

        if (receiverIt != mConfigReceivers.end()) {

            fprintf(out, "    -> received by %s, %s\n", receiverIt->second.first.c_str(),

                    receiverIt->second.second.c_str());

        }

        // TODO: Print the contents of the config too.

    }

}
@@ -227,7 +236,8 @@ StatsdConfig build_fake_config() { 
    metric->mutable_bucket()->set_bucket_size_millis(30 * 1000L);



    // Anomaly threshold for screen-on count.

    Alert* alert = config.add_alert();

    // TODO(b/70627390): Uncomment once the bug is fixed.

    /*Alert* alert = config.add_alert();

    alert->set_name("ALERT_1");

    alert->set_metric_name("METRIC_1");

    alert->set_number_of_buckets(6);
@@ -235,7 +245,7 @@ StatsdConfig build_fake_config() { 
    alert->set_refractory_period_secs(30);

    Alert::IncidentdDetails* details = alert->mutable_incidentd_details();

    details->add_section(12);

    details->add_section(13);

    details->add_section(13);*/



    // Count process state changes, slice by uid.

    metric = config.add_count_metric();
@@ -246,6 +256,8 @@ StatsdConfig build_fake_config() { 
    keyMatcher->set_key(UID_PROCESS_STATE_UID_KEY);



    // Anomaly threshold for background count.

    // TODO(b/70627390): Uncomment once the bug is fixed.

    /*

    alert = config.add_alert();

    alert->set_name("ALERT_2");

    alert->set_metric_name("METRIC_2");
@@ -254,7 +266,7 @@ StatsdConfig build_fake_config() { 
    alert->set_refractory_period_secs(20);

    details = alert->mutable_incidentd_details();

    details->add_section(14);

    details->add_section(15);

    details->add_section(15);*/



    // Count process state changes, slice by uid, while SCREEN_IS_OFF

    metric = config.add_count_metric();
@@ -326,6 +338,8 @@ StatsdConfig build_fake_config() { 
    durationMetric->set_what("SCREEN_IS_ON");



    // Anomaly threshold for background count.

    // TODO(b/70627390): Uncomment once the bug is fixed.

    /*

    alert = config.add_alert();

    alert->set_name("ALERT_8");

    alert->set_metric_name("METRIC_8");
@@ -333,7 +347,7 @@ StatsdConfig build_fake_config() { 
    alert->set_trigger_if_sum_gt(2000000000); // 2 seconds

    alert->set_refractory_period_secs(120);

    details = alert->mutable_incidentd_details();

    details->add_section(-1);

    details->add_section(-1);*/



    // Value metric to count KERNEL_WAKELOCK when screen turned on

    ValueMetric* valueMetric = config.add_value_metric();

cmds/statsd/src/config/ConfigManager.h

+10 −15
Original line number Diff line number Diff line
@@ -19,8 +19,9 @@ 
#include "config/ConfigKey.h"

#include "config/ConfigListener.h"



#include <map>

#include <set>

#include <string>

#include <unordered_map>



#include <stdio.h>
@@ -28,13 +29,7 @@ namespace android { 
namespace os {

namespace statsd {



using android::RefBase;

using std::string;

using std::unordered_map;

using std::vector;

using std::pair;



// Util function to Hard code a test metric for counting screen on events.

// Util function to build a hard coded config with test metrics.

StatsdConfig build_fake_config();



/**
@@ -43,7 +38,7 @@ StatsdConfig build_fake_config(); 
 * TODO: Store the configs persistently too.

 * TODO: Dump method for debugging.

 */

class ConfigManager : public virtual RefBase {

class ConfigManager : public virtual android::RefBase {

public:

    ConfigManager();

    virtual ~ConfigManager();
@@ -68,17 +63,17 @@ public: 
    /**

     * Sets the broadcast receiver for a configuration key.

     */

    void SetConfigReceiver(const ConfigKey& key, const string& pkg, const string& cls);

    void SetConfigReceiver(const ConfigKey& key, const std::string& pkg, const std::string& cls);



    /**

     * Returns the package name and class name representing the broadcast receiver for this config.

     */

    const pair<string, string> GetConfigReceiver(const ConfigKey& key) const;

    const std::pair<std::string, std::string> GetConfigReceiver(const ConfigKey& key) const;



    /**

     * Returns all config keys registered.

     */

    vector<ConfigKey> GetAllConfigKeys() const;

    std::vector<ConfigKey> GetAllConfigKeys() const;



    /**

     * Erase any broadcast receiver associated with this config key.
@@ -121,18 +116,18 @@ private: 
    /**

     * The Configs that have been set. Each config should

     */

    unordered_map<ConfigKey, StatsdConfig> mConfigs;

    std::set<ConfigKey> mConfigs;



    /**

     * Each config key can be subscribed by up to one receiver, specified as the package name and

     * class name.

     */

    unordered_map<ConfigKey, pair<string, string>> mConfigReceivers;

    std::map<ConfigKey, std::pair<std::string, std::string>> mConfigReceivers;



    /**

     * The ConfigListeners that will be told about changes.

     */

    vector<sp<ConfigListener>> mListeners;

    std::vector<sp<ConfigListener>> mListeners;

};



}  // namespace statsd

cmds/statsd/src/metrics/CountMetricProducer.cpp

+10 −11
Original line number Diff line number Diff line
@@ -66,7 +66,7 @@ CountMetricProducer::CountMetricProducer(const ConfigKey& key, const CountMetric 
                                         const int conditionIndex,

                                         const sp<ConditionWizard>& wizard,

                                         const uint64_t startTimeNs)

    : MetricProducer(key, startTimeNs, conditionIndex, wizard), mMetric(metric) {

    : MetricProducer(metric.name(), key, startTimeNs, conditionIndex, wizard) {

    // TODO: evaluate initial conditions. and set mConditionMet.

    if (metric.has_bucket() && metric.bucket().has_bucket_size_millis()) {

        mBucketSizeNs = metric.bucket().bucket_size_millis() * 1000 * 1000;
@@ -92,18 +92,18 @@ CountMetricProducer::~CountMetricProducer() { 
}



void CountMetricProducer::onSlicedConditionMayChangeLocked(const uint64_t eventTime) {

    VLOG("Metric %s onSlicedConditionMayChange", mMetric.name().c_str());

    VLOG("Metric %s onSlicedConditionMayChange", mName.c_str());

}



void CountMetricProducer::onDumpReportLocked(const uint64_t dumpTimeNs,

                                             ProtoOutputStream* protoOutput) {

    flushIfNeededLocked(dumpTimeNs);



    protoOutput->write(FIELD_TYPE_STRING | FIELD_ID_NAME, mMetric.name());

    protoOutput->write(FIELD_TYPE_STRING | FIELD_ID_NAME, mName);

    protoOutput->write(FIELD_TYPE_INT64 | FIELD_ID_START_REPORT_NANOS, (long long)mStartTimeNs);

    long long protoToken = protoOutput->start(FIELD_TYPE_MESSAGE | FIELD_ID_COUNT_METRICS);



    VLOG("metric %s dump report now...", mMetric.name().c_str());

    VLOG("metric %s dump report now...", mName.c_str());



    for (const auto& counter : mPastBuckets) {

        const HashableDimensionKey& hashableKey = counter.first;
@@ -160,7 +160,7 @@ void CountMetricProducer::onDumpReportLocked(const uint64_t dumpTimeNs, 


void CountMetricProducer::onConditionChangedLocked(const bool conditionMet,

                                                   const uint64_t eventTime) {

    VLOG("Metric %s onConditionChanged", mMetric.name().c_str());

    VLOG("Metric %s onConditionChanged", mName.c_str());

    mCondition = conditionMet;

}
@@ -172,11 +172,10 @@ bool CountMetricProducer::hitGuardRailLocked(const HashableDimensionKey& newKey) 
    // 1. Report the tuple count if the tuple count > soft limit

    if (mCurrentSlicedCounter->size() > StatsdStats::kDimensionKeySizeSoftLimit - 1) {

        size_t newTupleCount = mCurrentSlicedCounter->size() + 1;

        StatsdStats::getInstance().noteMetricDimensionSize(mConfigKey, mMetric.name(),

                                                           newTupleCount);

        StatsdStats::getInstance().noteMetricDimensionSize(mConfigKey, mName, newTupleCount);

        // 2. Don't add more tuples, we are above the allowed threshold. Drop the data.

        if (newTupleCount > StatsdStats::kDimensionKeySizeHardLimit) {

            ALOGE("CountMetric %s dropping data for dimension key %s", mMetric.name().c_str(),

            ALOGE("CountMetric %s dropping data for dimension key %s", mName.c_str(),

                  newKey.c_str());

            return true;

        }
@@ -218,7 +217,7 @@ void CountMetricProducer::onMatchedLogEventInternalLocked( 
                                         mCurrentSlicedCounter->find(eventKey)->second);

    }



    VLOG("metric %s %s->%lld", mMetric.name().c_str(), eventKey.c_str(),

    VLOG("metric %s %s->%lld", mName.c_str(), eventKey.c_str(),

         (long long)(*mCurrentSlicedCounter)[eventKey]);

}
@@ -237,7 +236,7 @@ void CountMetricProducer::flushIfNeededLocked(const uint64_t& eventTimeNs) { 
        info.mCount = counter.second;

        auto& bucketList = mPastBuckets[counter.first];

        bucketList.push_back(info);

        VLOG("metric %s, dump key value: %s -> %lld", mMetric.name().c_str(), counter.first.c_str(),

        VLOG("metric %s, dump key value: %s -> %lld", mName.c_str(), counter.first.c_str(),

             (long long)counter.second);

    }
@@ -250,7 +249,7 @@ void CountMetricProducer::flushIfNeededLocked(const uint64_t& eventTimeNs) { 
    uint64_t numBucketsForward = (eventTimeNs - mCurrentBucketStartTimeNs) / mBucketSizeNs;

    mCurrentBucketStartTimeNs = mCurrentBucketStartTimeNs + numBucketsForward * mBucketSizeNs;

    mCurrentBucketNum += numBucketsForward;

    VLOG("metric %s: new bucket start time: %lld", mMetric.name().c_str(),

    VLOG("metric %s: new bucket start time: %lld", mName.c_str(),

         (long long)mCurrentBucketStartTimeNs);

}

cmds/statsd/src/metrics/CountMetricProducer.h

+0 −2
Original line number Diff line number Diff line
@@ -76,8 +76,6 @@ private: 
    // Util function to flush the old packet.

    void flushIfNeededLocked(const uint64_t& newEventTime);



    const CountMetric mMetric;



    // TODO: Add a lock to mPastBuckets.

    std::unordered_map<HashableDimensionKey, std::vector<CountBucket>> mPastBuckets;

cmds/statsd/src/metrics/DurationMetricProducer.cpp

+11 −12
Original line number Diff line number Diff line
@@ -68,8 +68,8 @@ DurationMetricProducer::DurationMetricProducer(const ConfigKey& key, const Durat 
                                               const sp<ConditionWizard>& wizard,

                                               const vector<KeyMatcher>& internalDimension,

                                               const uint64_t startTimeNs)

    : MetricProducer(key, startTimeNs, conditionIndex, wizard),

      mMetric(metric),

    : MetricProducer(metric.name(), key, startTimeNs, conditionIndex, wizard),

      mAggregationType(metric.aggregation_type()),

      mStartIndex(startIndex),

      mStopIndex(stopIndex),

      mStopAllIndex(stopAllIndex),
@@ -114,20 +114,20 @@ sp<AnomalyTracker> DurationMetricProducer::createAnomalyTracker(const Alert &ale 


unique_ptr<DurationTracker> DurationMetricProducer::createDurationTracker(

        const HashableDimensionKey& eventKey) const {

    switch (mMetric.aggregation_type()) {

    switch (mAggregationType) {

        case DurationMetric_AggregationType_SUM:

            return make_unique<OringDurationTracker>(

                    mConfigKey, mMetric.name(), eventKey, mWizard, mConditionTrackerIndex, mNested,

                    mConfigKey, mName, eventKey, mWizard, mConditionTrackerIndex, mNested,

                    mCurrentBucketStartTimeNs, mBucketSizeNs, mAnomalyTrackers);

        case DurationMetric_AggregationType_MAX_SPARSE:

            return make_unique<MaxDurationTracker>(

                    mConfigKey, mMetric.name(), eventKey, mWizard, mConditionTrackerIndex, mNested,

                    mConfigKey, mName, eventKey, mWizard, mConditionTrackerIndex, mNested,

                    mCurrentBucketStartTimeNs, mBucketSizeNs, mAnomalyTrackers);

    }

}



void DurationMetricProducer::onSlicedConditionMayChangeLocked(const uint64_t eventTime) {

    VLOG("Metric %s onSlicedConditionMayChange", mMetric.name().c_str());

    VLOG("Metric %s onSlicedConditionMayChange", mName.c_str());

    flushIfNeededLocked(eventTime);

    // Now for each of the on-going event, check if the condition has changed for them.

    for (auto& pair : mCurrentSlicedDuration) {
@@ -137,7 +137,7 @@ void DurationMetricProducer::onSlicedConditionMayChangeLocked(const uint64_t eve 


void DurationMetricProducer::onConditionChangedLocked(const bool conditionMet,

                                                      const uint64_t eventTime) {

    VLOG("Metric %s onConditionChanged", mMetric.name().c_str());

    VLOG("Metric %s onConditionChanged", mName.c_str());

    mCondition = conditionMet;

    flushIfNeededLocked(eventTime);

    // TODO: need to populate the condition change time from the event which triggers the condition
@@ -151,11 +151,11 @@ void DurationMetricProducer::onDumpReportLocked(const uint64_t dumpTimeNs, 
                                                ProtoOutputStream* protoOutput) {

    flushIfNeededLocked(dumpTimeNs);



    protoOutput->write(FIELD_TYPE_STRING | FIELD_ID_NAME, mMetric.name());

    protoOutput->write(FIELD_TYPE_STRING | FIELD_ID_NAME, mName);

    protoOutput->write(FIELD_TYPE_INT64 | FIELD_ID_START_REPORT_NANOS, (long long)mStartTimeNs);

    long long protoToken = protoOutput->start(FIELD_TYPE_MESSAGE | FIELD_ID_DURATION_METRICS);



    VLOG("metric %s dump report now...", mMetric.name().c_str());

    VLOG("metric %s dump report now...", mName.c_str());



    for (const auto& pair : mPastBuckets) {

        const HashableDimensionKey& hashableKey = pair.first;
@@ -236,11 +236,10 @@ bool DurationMetricProducer::hitGuardRailLocked(const HashableDimensionKey& newK 
    // 1. Report the tuple count if the tuple count > soft limit

    if (mCurrentSlicedDuration.size() > StatsdStats::kDimensionKeySizeSoftLimit - 1) {

        size_t newTupleCount = mCurrentSlicedDuration.size() + 1;

        StatsdStats::getInstance().noteMetricDimensionSize(mConfigKey, mMetric.name(),

                                                           newTupleCount);

        StatsdStats::getInstance().noteMetricDimensionSize(mConfigKey, mName, newTupleCount);

        // 2. Don't add more tuples, we are above the allowed threshold. Drop the data.

        if (newTupleCount > StatsdStats::kDimensionKeySizeHardLimit) {

            ALOGE("DurationMetric %s dropping data for dimension key %s", mMetric.name().c_str(),

            ALOGE("DurationMetric %s dropping data for dimension key %s", mName.c_str(),

                  newKey.c_str());

            return true;

        }