Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 0630ad6d authored by TreeHugger Robot's avatar TreeHugger Robot Committed by Android (Google) Code Review
Browse files

Merge "Further reduce statsd memory usage."

parents 157adc3b f09569f8

cmds/statsd/src/config/ConfigManager.cpp

+36 −22
Original line number Diff line number Diff line
@@ -29,6 +29,12 @@ namespace android { 
namespace os {

namespace statsd {



using std::map;

using std::pair;

using std::set;

using std::string;

using std::vector;



#define STATS_SERVICE_DIR "/data/misc/stats-service"



using android::base::StringPrintf;
@@ -41,8 +47,14 @@ ConfigManager::~ConfigManager() { 
}



void ConfigManager::Startup() {

    StorageManager::readConfigFromDisk(mConfigs);



    map<ConfigKey, StatsdConfig> configsFromDisk;

    StorageManager::readConfigFromDisk(configsFromDisk);

    // TODO(b/70667694): Make the configs from disk be used. And remove the fake config,

    // and tests shouldn't call this Startup(), maybe call StartupForTest() so we don't read

    // configs from disk for tests.

    // for (const auto& pair : configsFromDisk) {

    //    UpdateConfig(pair.first, pair.second);

    //}

    // this should be called from StatsService when it receives a statsd_config

    UpdateConfig(ConfigKey(1000, "fake"), build_fake_config());

}
@@ -52,9 +64,8 @@ void ConfigManager::AddListener(const sp<ConfigListener>& listener) { 
}



void ConfigManager::UpdateConfig(const ConfigKey& key, const StatsdConfig& config) {

    // Add to map

    mConfigs[key] = config;

    // Why doesn't this work? mConfigs.insert({key, config});

    // Add to set

    mConfigs.insert(key);



    // Save to disk

    update_saved_configs(key, config);
@@ -74,7 +85,7 @@ void ConfigManager::RemoveConfigReceiver(const ConfigKey& key) { 
}



void ConfigManager::RemoveConfig(const ConfigKey& key) {

    unordered_map<ConfigKey, StatsdConfig>::iterator it = mConfigs.find(key);

    auto it = mConfigs.find(key);

    if (it != mConfigs.end()) {

        // Remove from map

        mConfigs.erase(it);
@@ -100,9 +111,9 @@ void ConfigManager::RemoveConfigs(int uid) { 


    for (auto it = mConfigs.begin(); it != mConfigs.end();) {

        // Remove from map

        if (it->first.GetUid() == uid) {

            removed.push_back(it->first);

            mConfigReceivers.erase(it->first);

        if (it->GetUid() == uid) {

            removed.push_back(*it);

            mConfigReceivers.erase(*it);

            it = mConfigs.erase(it);

        } else {

            it++;
@@ -123,10 +134,10 @@ void ConfigManager::RemoveAllConfigs() { 


    for (auto it = mConfigs.begin(); it != mConfigs.end();) {

        // Remove from map

        removed.push_back(it->first);

        auto receiverIt = mConfigReceivers.find(it->first);

        removed.push_back(*it);

        auto receiverIt = mConfigReceivers.find(*it);

        if (receiverIt != mConfigReceivers.end()) {

            mConfigReceivers.erase(it->first);

            mConfigReceivers.erase(*it);

        }

        it = mConfigs.erase(it);

    }
@@ -143,7 +154,7 @@ void ConfigManager::RemoveAllConfigs() { 
vector<ConfigKey> ConfigManager::GetAllConfigKeys() const {

    vector<ConfigKey> ret;

    for (auto it = mConfigs.cbegin(); it != mConfigs.cend(); ++it) {

        ret.push_back(it->first);

        ret.push_back(*it);

    }

    return ret;

}
@@ -160,15 +171,13 @@ const pair<string, string> ConfigManager::GetConfigReceiver(const ConfigKey& key 
void ConfigManager::Dump(FILE* out) {

    fprintf(out, "CONFIGURATIONS (%d)\n", (int)mConfigs.size());

    fprintf(out, "     uid name\n");

    for (unordered_map<ConfigKey, StatsdConfig>::const_iterator it = mConfigs.begin();

         it != mConfigs.end(); it++) {

        fprintf(out, "  %6d %s\n", it->first.GetUid(), it->first.GetName().c_str());

        auto receiverIt = mConfigReceivers.find(it->first);

    for (const auto& key : mConfigs) {

        fprintf(out, "  %6d %s\n", key.GetUid(), key.GetName().c_str());

        auto receiverIt = mConfigReceivers.find(key);

        if (receiverIt != mConfigReceivers.end()) {

            fprintf(out, "    -> received by %s, %s\n", receiverIt->second.first.c_str(),

                    receiverIt->second.second.c_str());

        }

        // TODO: Print the contents of the config too.

    }

}
@@ -227,7 +236,8 @@ StatsdConfig build_fake_config() { 
    metric->mutable_bucket()->set_bucket_size_millis(30 * 1000L);



    // Anomaly threshold for screen-on count.

    Alert* alert = config.add_alert();

    // TODO(b/70627390): Uncomment once the bug is fixed.

    /*Alert* alert = config.add_alert();

    alert->set_name("ALERT_1");

    alert->set_metric_name("METRIC_1");

    alert->set_number_of_buckets(6);
@@ -235,7 +245,7 @@ StatsdConfig build_fake_config() { 
    alert->set_refractory_period_secs(30);

    Alert::IncidentdDetails* details = alert->mutable_incidentd_details();

    details->add_section(12);

    details->add_section(13);

    details->add_section(13);*/



    // Count process state changes, slice by uid.

    metric = config.add_count_metric();
@@ -246,6 +256,8 @@ StatsdConfig build_fake_config() { 
    keyMatcher->set_key(UID_PROCESS_STATE_UID_KEY);



    // Anomaly threshold for background count.

    // TODO(b/70627390): Uncomment once the bug is fixed.

    /*

    alert = config.add_alert();

    alert->set_name("ALERT_2");

    alert->set_metric_name("METRIC_2");
@@ -254,7 +266,7 @@ StatsdConfig build_fake_config() { 
    alert->set_refractory_period_secs(20);

    details = alert->mutable_incidentd_details();

    details->add_section(14);

    details->add_section(15);

    details->add_section(15);*/



    // Count process state changes, slice by uid, while SCREEN_IS_OFF

    metric = config.add_count_metric();
@@ -326,6 +338,8 @@ StatsdConfig build_fake_config() { 
    durationMetric->set_what("SCREEN_IS_ON");



    // Anomaly threshold for background count.

    // TODO(b/70627390): Uncomment once the bug is fixed.

    /*

    alert = config.add_alert();

    alert->set_name("ALERT_8");

    alert->set_metric_name("METRIC_8");
@@ -333,7 +347,7 @@ StatsdConfig build_fake_config() { 
    alert->set_trigger_if_sum_gt(2000000000); // 2 seconds

    alert->set_refractory_period_secs(120);

    details = alert->mutable_incidentd_details();

    details->add_section(-1);

    details->add_section(-1);*/



    // Value metric to count KERNEL_WAKELOCK when screen turned on

    ValueMetric* valueMetric = config.add_value_metric();

cmds/statsd/src/config/ConfigManager.h

+10 −15
Original line number Diff line number Diff line
@@ -19,8 +19,9 @@ 
#include "config/ConfigKey.h"

#include "config/ConfigListener.h"



#include <map>

#include <set>

#include <string>

#include <unordered_map>



#include <stdio.h>
@@ -28,13 +29,7 @@ namespace android { 
namespace os {

namespace statsd {



using android::RefBase;

using std::string;

using std::unordered_map;

using std::vector;

using std::pair;



// Util function to Hard code a test metric for counting screen on events.

// Util function to build a hard coded config with test metrics.

StatsdConfig build_fake_config();



/**
@@ -43,7 +38,7 @@ StatsdConfig build_fake_config(); 
 * TODO: Store the configs persistently too.

 * TODO: Dump method for debugging.

 */

class ConfigManager : public virtual RefBase {

class ConfigManager : public virtual android::RefBase {

public:

    ConfigManager();

    virtual ~ConfigManager();
@@ -68,17 +63,17 @@ public: 
    /**

     * Sets the broadcast receiver for a configuration key.

     */

    void SetConfigReceiver(const ConfigKey& key, const string& pkg, const string& cls);

    void SetConfigReceiver(const ConfigKey& key, const std::string& pkg, const std::string& cls);



    /**

     * Returns the package name and class name representing the broadcast receiver for this config.

     */

    const pair<string, string> GetConfigReceiver(const ConfigKey& key) const;

    const std::pair<std::string, std::string> GetConfigReceiver(const ConfigKey& key) const;



    /**

     * Returns all config keys registered.

     */

    vector<ConfigKey> GetAllConfigKeys() const;

    std::vector<ConfigKey> GetAllConfigKeys() const;



    /**

     * Erase any broadcast receiver associated with this config key.
@@ -121,18 +116,18 @@ private: 
    /**

     * The Configs that have been set. Each config should

     */

    unordered_map<ConfigKey, StatsdConfig> mConfigs;

    std::set<ConfigKey> mConfigs;



    /**

     * Each config key can be subscribed by up to one receiver, specified as the package name and

     * class name.

     */

    unordered_map<ConfigKey, pair<string, string>> mConfigReceivers;

    std::map<ConfigKey, std::pair<std::string, std::string>> mConfigReceivers;



    /**

     * The ConfigListeners that will be told about changes.

     */

    vector<sp<ConfigListener>> mListeners;

    std::vector<sp<ConfigListener>> mListeners;

};



}  // namespace statsd

cmds/statsd/src/metrics/CountMetricProducer.cpp

+10 −11
Original line number Diff line number Diff line
@@ -66,7 +66,7 @@ CountMetricProducer::CountMetricProducer(const ConfigKey& key, const CountMetric 
                                         const int conditionIndex,

                                         const sp<ConditionWizard>& wizard,

                                         const uint64_t startTimeNs)

    : MetricProducer(key, startTimeNs, conditionIndex, wizard), mMetric(metric) {

    : MetricProducer(metric.name(), key, startTimeNs, conditionIndex, wizard) {

    // TODO: evaluate initial conditions. and set mConditionMet.

    if (metric.has_bucket() && metric.bucket().has_bucket_size_millis()) {

        mBucketSizeNs = metric.bucket().bucket_size_millis() * 1000 * 1000;
@@ -92,18 +92,18 @@ CountMetricProducer::~CountMetricProducer() { 
}



void CountMetricProducer::onSlicedConditionMayChangeLocked(const uint64_t eventTime) {

    VLOG("Metric %s onSlicedConditionMayChange", mMetric.name().c_str());

    VLOG("Metric %s onSlicedConditionMayChange", mName.c_str());

}



void CountMetricProducer::onDumpReportLocked(const uint64_t dumpTimeNs,

                                             ProtoOutputStream* protoOutput) {

    flushIfNeededLocked(dumpTimeNs);



    protoOutput->write(FIELD_TYPE_STRING | FIELD_ID_NAME, mMetric.name());

    protoOutput->write(FIELD_TYPE_STRING | FIELD_ID_NAME, mName);

    protoOutput->write(FIELD_TYPE_INT64 | FIELD_ID_START_REPORT_NANOS, (long long)mStartTimeNs);

    long long protoToken = protoOutput->start(FIELD_TYPE_MESSAGE | FIELD_ID_COUNT_METRICS);



    VLOG("metric %s dump report now...", mMetric.name().c_str());

    VLOG("metric %s dump report now...", mName.c_str());



    for (const auto& counter : mPastBuckets) {

        const HashableDimensionKey& hashableKey = counter.first;
@@ -160,7 +160,7 @@ void CountMetricProducer::onDumpReportLocked(const uint64_t dumpTimeNs, 


void CountMetricProducer::onConditionChangedLocked(const bool conditionMet,

                                                   const uint64_t eventTime) {

    VLOG("Metric %s onConditionChanged", mMetric.name().c_str());

    VLOG("Metric %s onConditionChanged", mName.c_str());

    mCondition = conditionMet;

}
@@ -172,11 +172,10 @@ bool CountMetricProducer::hitGuardRailLocked(const HashableDimensionKey& newKey) 
    // 1. Report the tuple count if the tuple count > soft limit

    if (mCurrentSlicedCounter->size() > StatsdStats::kDimensionKeySizeSoftLimit - 1) {

        size_t newTupleCount = mCurrentSlicedCounter->size() + 1;

        StatsdStats::getInstance().noteMetricDimensionSize(mConfigKey, mMetric.name(),

                                                           newTupleCount);

        StatsdStats::getInstance().noteMetricDimensionSize(mConfigKey, mName, newTupleCount);

        // 2. Don't add more tuples, we are above the allowed threshold. Drop the data.

        if (newTupleCount > StatsdStats::kDimensionKeySizeHardLimit) {

            ALOGE("CountMetric %s dropping data for dimension key %s", mMetric.name().c_str(),

            ALOGE("CountMetric %s dropping data for dimension key %s", mName.c_str(),

                  newKey.c_str());

            return true;

        }
@@ -218,7 +217,7 @@ void CountMetricProducer::onMatchedLogEventInternalLocked( 
                                         mCurrentSlicedCounter->find(eventKey)->second);

    }



    VLOG("metric %s %s->%lld", mMetric.name().c_str(), eventKey.c_str(),

    VLOG("metric %s %s->%lld", mName.c_str(), eventKey.c_str(),

         (long long)(*mCurrentSlicedCounter)[eventKey]);

}
@@ -237,7 +236,7 @@ void CountMetricProducer::flushIfNeededLocked(const uint64_t& eventTimeNs) { 
        info.mCount = counter.second;

        auto& bucketList = mPastBuckets[counter.first];

        bucketList.push_back(info);

        VLOG("metric %s, dump key value: %s -> %lld", mMetric.name().c_str(), counter.first.c_str(),

        VLOG("metric %s, dump key value: %s -> %lld", mName.c_str(), counter.first.c_str(),

             (long long)counter.second);

    }
@@ -250,7 +249,7 @@ void CountMetricProducer::flushIfNeededLocked(const uint64_t& eventTimeNs) { 
    uint64_t numBucketsForward = (eventTimeNs - mCurrentBucketStartTimeNs) / mBucketSizeNs;

    mCurrentBucketStartTimeNs = mCurrentBucketStartTimeNs + numBucketsForward * mBucketSizeNs;

    mCurrentBucketNum += numBucketsForward;

    VLOG("metric %s: new bucket start time: %lld", mMetric.name().c_str(),

    VLOG("metric %s: new bucket start time: %lld", mName.c_str(),

         (long long)mCurrentBucketStartTimeNs);

}

cmds/statsd/src/metrics/CountMetricProducer.h

+0 −2
Original line number Diff line number Diff line
@@ -76,8 +76,6 @@ private: 
    // Util function to flush the old packet.

    void flushIfNeededLocked(const uint64_t& newEventTime);



    const CountMetric mMetric;



    // TODO: Add a lock to mPastBuckets.

    std::unordered_map<HashableDimensionKey, std::vector<CountBucket>> mPastBuckets;

cmds/statsd/src/metrics/DurationMetricProducer.cpp

+11 −12
Original line number Diff line number Diff line
@@ -68,8 +68,8 @@ DurationMetricProducer::DurationMetricProducer(const ConfigKey& key, const Durat 
                                               const sp<ConditionWizard>& wizard,

                                               const vector<KeyMatcher>& internalDimension,

                                               const uint64_t startTimeNs)

    : MetricProducer(key, startTimeNs, conditionIndex, wizard),

      mMetric(metric),

    : MetricProducer(metric.name(), key, startTimeNs, conditionIndex, wizard),

      mAggregationType(metric.aggregation_type()),

      mStartIndex(startIndex),

      mStopIndex(stopIndex),

      mStopAllIndex(stopAllIndex),
@@ -114,20 +114,20 @@ sp<AnomalyTracker> DurationMetricProducer::createAnomalyTracker(const Alert &ale 


unique_ptr<DurationTracker> DurationMetricProducer::createDurationTracker(

        const HashableDimensionKey& eventKey) const {

    switch (mMetric.aggregation_type()) {

    switch (mAggregationType) {

        case DurationMetric_AggregationType_SUM:

            return make_unique<OringDurationTracker>(

                    mConfigKey, mMetric.name(), eventKey, mWizard, mConditionTrackerIndex, mNested,

                    mConfigKey, mName, eventKey, mWizard, mConditionTrackerIndex, mNested,

                    mCurrentBucketStartTimeNs, mBucketSizeNs, mAnomalyTrackers);

        case DurationMetric_AggregationType_MAX_SPARSE:

            return make_unique<MaxDurationTracker>(

                    mConfigKey, mMetric.name(), eventKey, mWizard, mConditionTrackerIndex, mNested,

                    mConfigKey, mName, eventKey, mWizard, mConditionTrackerIndex, mNested,

                    mCurrentBucketStartTimeNs, mBucketSizeNs, mAnomalyTrackers);

    }

}



void DurationMetricProducer::onSlicedConditionMayChangeLocked(const uint64_t eventTime) {

    VLOG("Metric %s onSlicedConditionMayChange", mMetric.name().c_str());

    VLOG("Metric %s onSlicedConditionMayChange", mName.c_str());

    flushIfNeededLocked(eventTime);

    // Now for each of the on-going event, check if the condition has changed for them.

    for (auto& pair : mCurrentSlicedDuration) {
@@ -137,7 +137,7 @@ void DurationMetricProducer::onSlicedConditionMayChangeLocked(const uint64_t eve 


void DurationMetricProducer::onConditionChangedLocked(const bool conditionMet,

                                                      const uint64_t eventTime) {

    VLOG("Metric %s onConditionChanged", mMetric.name().c_str());

    VLOG("Metric %s onConditionChanged", mName.c_str());

    mCondition = conditionMet;

    flushIfNeededLocked(eventTime);

    // TODO: need to populate the condition change time from the event which triggers the condition
@@ -151,11 +151,11 @@ void DurationMetricProducer::onDumpReportLocked(const uint64_t dumpTimeNs, 
                                                ProtoOutputStream* protoOutput) {

    flushIfNeededLocked(dumpTimeNs);



    protoOutput->write(FIELD_TYPE_STRING | FIELD_ID_NAME, mMetric.name());

    protoOutput->write(FIELD_TYPE_STRING | FIELD_ID_NAME, mName);

    protoOutput->write(FIELD_TYPE_INT64 | FIELD_ID_START_REPORT_NANOS, (long long)mStartTimeNs);

    long long protoToken = protoOutput->start(FIELD_TYPE_MESSAGE | FIELD_ID_DURATION_METRICS);



    VLOG("metric %s dump report now...", mMetric.name().c_str());

    VLOG("metric %s dump report now...", mName.c_str());



    for (const auto& pair : mPastBuckets) {

        const HashableDimensionKey& hashableKey = pair.first;
@@ -236,11 +236,10 @@ bool DurationMetricProducer::hitGuardRailLocked(const HashableDimensionKey& newK 
    // 1. Report the tuple count if the tuple count > soft limit

    if (mCurrentSlicedDuration.size() > StatsdStats::kDimensionKeySizeSoftLimit - 1) {

        size_t newTupleCount = mCurrentSlicedDuration.size() + 1;

        StatsdStats::getInstance().noteMetricDimensionSize(mConfigKey, mMetric.name(),

                                                           newTupleCount);

        StatsdStats::getInstance().noteMetricDimensionSize(mConfigKey, mName, newTupleCount);

        // 2. Don't add more tuples, we are above the allowed threshold. Drop the data.

        if (newTupleCount > StatsdStats::kDimensionKeySizeHardLimit) {

            ALOGE("DurationMetric %s dropping data for dimension key %s", mMetric.name().c_str(),

            ALOGE("DurationMetric %s dropping data for dimension key %s", mName.c_str(),

                  newKey.c_str());

            return true;

        }