Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit b356151e authored by Yao Chen's avatar Yao Chen
Browse files

Add StatsdStats and guardrail. 

+ StatsdStats is the global class that tracks the stats about statsd.

+ Added guardrail for classes that have a map which could potentially grow
  unboundedly with the number of logs.

TODO: add unit tests & CTS for StatsdStats, and guardrail
      add stats for pulled atoms.

Test: statsd_test

Change-Id: I0ea562de4dd3f6162f7923a9c193420b482c1d51
parent 16fd2e9a

cmds/statsd/Android.mk

+4 −2
Original line number Diff line number Diff line
@@ -57,7 +57,8 @@ statsd_common_src := \ 
    src/StatsLogProcessor.cpp \

    src/StatsService.cpp \

    src/stats_util.cpp \

    src/guardrail/MemoryLeakTrackUtil.cpp

    src/guardrail/MemoryLeakTrackUtil.cpp \

    src/guardrail/StatsdStats.cpp



statsd_common_c_includes := \

    $(LOCAL_PATH)/src \
@@ -167,7 +168,8 @@ LOCAL_SRC_FILES := \ 
    tests/metrics/MaxDurationTracker_test.cpp \

    tests/metrics/CountMetricProducer_test.cpp \

    tests/metrics/EventMetricProducer_test.cpp \

    tests/metrics/ValueMetricProducer_test.cpp

    tests/metrics/ValueMetricProducer_test.cpp \

    tests/guardrail/StatsdStats_test.cpp



LOCAL_STATIC_LIBRARIES := \

    libgmock

cmds/statsd/src/StatsLogProcessor.cpp

+16 −7
Original line number Diff line number Diff line
@@ -14,6 +14,7 @@ 
 * limitations under the License.

 */



#define DEBUG true  // STOPSHIP if true

#include "Log.h"

#include "statslog.h"
@@ -21,6 +22,7 @@ 
#include <dirent.h>

#include "StatsLogProcessor.h"

#include "android-base/stringprintf.h"

#include "guardrail/StatsdStats.h"

#include "metrics/CountMetricProducer.h"

#include "stats_util.h"

#include "storage/StorageManager.h"
@@ -82,6 +84,7 @@ void StatsLogProcessor::onAnomalyAlarmFired( 


// TODO: what if statsd service restarts? How do we know what logs are already processed before?

void StatsLogProcessor::OnLogEvent(const LogEvent& msg) {

    StatsdStats::getInstance().noteAtomLogged(msg.GetTagId(), msg.GetTimestampNs() / NS_PER_SEC);

    // pass the event to metrics managers.

    for (auto& pair : mMetricsManagers) {

        pair.second->onLogEvent(msg);
@@ -106,23 +109,26 @@ void StatsLogProcessor::OnLogEvent(const LogEvent& msg) { 
}



void StatsLogProcessor::OnConfigUpdated(const ConfigKey& key, const StatsdConfig& config) {

    ALOGD("Updated configuration for key %s", key.ToString().c_str());

    unique_ptr<MetricsManager> newMetricsManager = std::make_unique<MetricsManager>(key, config);



    auto it = mMetricsManagers.find(key);

    if (it != mMetricsManagers.end()) {

        it->second->finish();

    } else if (mMetricsManagers.size() > StatsdStats::kMaxConfigCount) {

        ALOGE("Can't accept more configs!");

        return;

    }



    ALOGD("Updated configuration for key %s", key.ToString().c_str());



    unique_ptr<MetricsManager> newMetricsManager = std::make_unique<MetricsManager>(config);

    if (newMetricsManager->isConfigValid()) {

        mUidMap->OnConfigUpdated(key);

        newMetricsManager->setAnomalyMonitor(mAnomalyMonitor);

        mMetricsManagers[key] = std::move(newMetricsManager);

        // Why doesn't this work? mMetricsManagers.insert({key, std::move(newMetricsManager)});

        ALOGD("StatsdConfig valid");

        VLOG("StatsdConfig valid");

    } else {

        // If there is any error in the config, don't use it.

        ALOGD("StatsdConfig NOT valid");

        ALOGE("StatsdConfig NOT valid");

    }

}
@@ -204,6 +210,7 @@ void StatsLogProcessor::OnConfigRemoved(const ConfigKey& key) { 
        mMetricsManagers.erase(it);

        mUidMap->OnConfigRemoved(key);

    }

    StatsdStats::getInstance().noteConfigRemoved(key);



    std::lock_guard<std::mutex> lock(mBroadcastTimesMutex);

    mLastBroadcastTimes.erase(key);
@@ -223,12 +230,14 @@ void StatsLogProcessor::flushIfNecessary(uint64_t timestampNs, 
            }

        }

        mLastBroadcastTimes[key] = timestampNs;

        ALOGD("StatsD requesting broadcast for %s", key.ToString().c_str());

        VLOG("StatsD requesting broadcast for %s", key.ToString().c_str());

        mSendBroadcast(key);

        StatsdStats::getInstance().noteBroadcastSent(key);

    } else if (totalBytes > kMaxSerializedBytes) { // Too late. We need to start clearing data.

        // We ignore the return value so we force each metric producer to clear its contents.

        metricsManager->onDumpReport();

        ALOGD("StatsD had to toss out metrics for %s", key.ToString().c_str());

        StatsdStats::getInstance().noteDataDrop(key);

        VLOG("StatsD had to toss out metrics for %s", key.ToString().c_str());

    }

}

cmds/statsd/src/StatsService.cpp

+13 −3
Original line number Diff line number Diff line
@@ -22,6 +22,7 @@ 
#include "config/ConfigKey.h"

#include "config/ConfigManager.h"

#include "guardrail/MemoryLeakTrackUtil.h"

#include "guardrail/StatsdStats.h"

#include "storage/DropboxReader.h"

#include "storage/StorageManager.h"
@@ -222,7 +223,7 @@ status_t StatsService::command(FILE* in, FILE* out, FILE* err, Vector<String8>& 
        }



        if (!args[0].compare(String8("print-stats"))) {

            return cmd_print_stats(out);

            return cmd_print_stats(out, args);

        }



        if (!args[0].compare(String8("clear-config"))) {
@@ -305,8 +306,9 @@ void StatsService::print_cmd_help(FILE* out) { 
    fprintf(out, "  NAME          The name of the configuration\n");

    fprintf(out, "\n");

    fprintf(out, "\n");

    fprintf(out, "usage: adb shell cmd stats print-stats\n");

    fprintf(out, "usage: adb shell cmd stats print-stats [reset]\n");

    fprintf(out, "  Prints some basic stats.\n");

    fprintf(out, "  reset: 1 to reset the statsd stats. default=0.\n");

}



status_t StatsService::cmd_trigger_broadcast(FILE* out, Vector<String8>& args) {
@@ -474,12 +476,20 @@ status_t StatsService::cmd_dump_report(FILE* out, FILE* err, const Vector<String 
    }

}



status_t StatsService::cmd_print_stats(FILE* out) {

status_t StatsService::cmd_print_stats(FILE* out, const Vector<String8>& args) {

    vector<ConfigKey> configs = mConfigManager->GetAllConfigKeys();

    for (const ConfigKey& key : configs) {

        fprintf(out, "Config %s uses %zu bytes\n", key.ToString().c_str(),

                mProcessor->GetMetricsSize(key));

    }

    fprintf(out, "Detailed statsd stats in logcat...");

    StatsdStats& statsdStats = StatsdStats::getInstance();

    bool reset = false;

    if (args.size() > 1) {

        reset = strtol(args[1].string(), NULL, 10);

    }

    vector<int8_t> output;

    statsdStats.dumpStats(&output, reset);

    return NO_ERROR;

}

cmds/statsd/src/StatsService.h

+1 −1
Original line number Diff line number Diff line
@@ -134,7 +134,7 @@ private: 
    /**

     * Prints some basic stats to std out.

     */

    status_t cmd_print_stats(FILE* out);

    status_t cmd_print_stats(FILE* out, const Vector<String8>& args);



    /**

     * Print the event log.

cmds/statsd/src/condition/SimpleConditionTracker.cpp

+28 −2
Original line number Diff line number Diff line
@@ -18,6 +18,7 @@ 
#include "Log.h"



#include "SimpleConditionTracker.h"

#include "guardrail/StatsdStats.h"



#include <log/logprint.h>
@@ -32,9 +33,10 @@ using std::unordered_map; 
using std::vector;



SimpleConditionTracker::SimpleConditionTracker(

        const string& name, const int index, const SimpleCondition& simpleCondition,

        const ConfigKey& key, const string& name, const int index,

        const SimpleCondition& simpleCondition,

        const unordered_map<string, int>& trackerNameIndexMap)

    : ConditionTracker(name, index) {

    : ConditionTracker(name, index), mConfigKey(key) {

    VLOG("creating SimpleConditionTracker %s", mName.c_str());

    mCountNesting = simpleCondition.count_nesting();
@@ -126,6 +128,24 @@ void SimpleConditionTracker::handleStopAll(std::vector<ConditionState>& conditio 
    conditionCache[mIndex] = ConditionState::kFalse;

}



bool SimpleConditionTracker::hitGuardRail(const HashableDimensionKey& newKey) {

    if (!mSliced || mSlicedConditionState.find(newKey) != mSlicedConditionState.end()) {

        // if the condition is not sliced or the key is not new, we are good!

        return false;

    }

    // 1. Report the tuple count if the tuple count > soft limit

    if (mSlicedConditionState.size() > StatsdStats::kDimensionKeySizeSoftLimit - 1) {

        size_t newTupleCount = mSlicedConditionState.size() + 1;

        StatsdStats::getInstance().noteConditionDimensionSize(mConfigKey, mName, newTupleCount);

        // 2. Don't add more tuples, we are above the allowed threshold. Drop the data.

        if (newTupleCount > StatsdStats::kDimensionKeySizeHardLimit) {

            ALOGE("Condition %s dropping data for dimension key %s", mName.c_str(), newKey.c_str());

            return true;

        }

    }

    return false;

}



void SimpleConditionTracker::handleConditionEvent(const HashableDimensionKey& outputKey,

                                                  bool matchStart,

                                                  std::vector<ConditionState>& conditionCache,
@@ -133,6 +153,12 @@ void SimpleConditionTracker::handleConditionEvent(const HashableDimensionKey& ou 
    bool changed = false;

    auto outputIt = mSlicedConditionState.find(outputKey);

    ConditionState newCondition;

    if (hitGuardRail(outputKey)) {

        conditionChangedCache[mIndex] = false;

        // Tells the caller it's evaluated.

        conditionCache[mIndex] = ConditionState::kUnknown;

        return;

    }

    if (outputIt == mSlicedConditionState.end()) {

        // We get a new output key.

        newCondition = matchStart ? ConditionState::kTrue : ConditionState::kFalse;