Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit faa1af53 authored by David Chen's avatar David Chen
Browse files

Includes annotations with statsd reports. 

It's tricky to determine the source of the metrics on a device
currently since we can take the union of multiple configs and send
only one giant statsd_config into statsd. We will use the int64 field
to track the sub config id's and the int32 field to track the version
for each sub config, but the fields are named more generically as
annotations.

The annotations are available in both the reports and metadata.

Test: Check that all unit-tests pass on marlin-eng
Bug: 77327261
Change-Id: Ic37c549c8b2991676f69948c515156765c9f5108
parent a24ddf58

cmds/statsd/src/guardrail/StatsdStats.cpp

+25 −2
Original line number Diff line number Diff line
@@ -77,6 +77,9 @@ const int FIELD_ID_CONFIG_STATS_CONDITION_STATS = 14; 
const int FIELD_ID_CONFIG_STATS_METRIC_STATS = 15;

const int FIELD_ID_CONFIG_STATS_ALERT_STATS = 16;

const int FIELD_ID_CONFIG_STATS_METRIC_DIMENSION_IN_CONDITION_STATS = 17;

const int FIELD_ID_CONFIG_STATS_ANNOTATION = 18;

const int FIELD_ID_CONFIG_STATS_ANNOTATION_INT64 = 1;

const int FIELD_ID_CONFIG_STATS_ANNOTATION_INT32 = 2;



const int FIELD_ID_MATCHER_STATS_ID = 1;

const int FIELD_ID_MATCHER_STATS_COUNT = 2;
@@ -116,8 +119,10 @@ void StatsdStats::addToIceBoxLocked(shared_ptr<ConfigStats>& stats) { 
    mIceBox.push_back(stats);

}



void StatsdStats::noteConfigReceived(const ConfigKey& key, int metricsCount, int conditionsCount,

                                     int matchersCount, int alertsCount, bool isValid) {

void StatsdStats::noteConfigReceived(

        const ConfigKey& key, int metricsCount, int conditionsCount, int matchersCount,

        int alertsCount, const std::list<std::pair<const int64_t, const int32_t>>& annotations,

        bool isValid) {

    lock_guard<std::mutex> lock(mLock);

    int32_t nowTimeSec = getWallClockSec();
@@ -133,6 +138,9 @@ void StatsdStats::noteConfigReceived(const ConfigKey& key, int metricsCount, int 
    configStats->matcher_count = matchersCount;

    configStats->alert_count = alertsCount;

    configStats->is_valid = isValid;

    for (auto& v : annotations) {

        configStats->annotations.emplace_back(v);

    }



    if (isValid) {

        mConfigStats[key] = configStats;
@@ -351,6 +359,7 @@ void StatsdStats::resetInternalLocked() { 
        config.second->broadcast_sent_time_sec.clear();

        config.second->data_drop_time_sec.clear();

        config.second->dump_report_time_sec.clear();

        config.second->annotations.clear();

        config.second->matcher_stats.clear();

        config.second->condition_stats.clear();

        config.second->metric_stats.clear();
@@ -394,6 +403,11 @@ void StatsdStats::dumpStats(FILE* out) const { 
                configStats->deletion_time_sec, configStats->metric_count,

                configStats->condition_count, configStats->matcher_count, configStats->alert_count,

                configStats->is_valid);

        for (const auto& annotation : configStats->annotations) {

            fprintf(out, "\tannotation: %lld, %d\n", (long long)annotation.first,

                    annotation.second);

        }



        for (const auto& broadcastTime : configStats->broadcast_sent_time_sec) {

            fprintf(out, "\tbroadcast time: %d\n", broadcastTime);

        }
@@ -497,6 +511,15 @@ void addConfigStatsToProto(const ConfigStats& configStats, ProtoOutputStream* pr 
                     dump);

    }



    for (const auto& annotation : configStats.annotations) {

        uint64_t token = proto->start(FIELD_TYPE_MESSAGE | FIELD_COUNT_REPEATED |

                                      FIELD_ID_CONFIG_STATS_ANNOTATION);

        proto->write(FIELD_TYPE_INT64 | FIELD_ID_CONFIG_STATS_ANNOTATION_INT64,

                     (long long)annotation.first);

        proto->write(FIELD_TYPE_INT32 | FIELD_ID_CONFIG_STATS_ANNOTATION_INT32, annotation.second);

        proto->end(token);

    }



    for (const auto& pair : configStats.matcher_stats) {

        uint64_t tmpToken = proto->start(FIELD_TYPE_MESSAGE | FIELD_COUNT_REPEATED |

                                          FIELD_ID_CONFIG_STATS_MATCHER_STATS);

cmds/statsd/src/guardrail/StatsdStats.h

+6 −1
Original line number Diff line number Diff line
@@ -66,6 +66,9 @@ struct ConfigStats { 
    // Stores the number of times an anomaly detection alert has been declared.

    // The map size is capped by kMaxConfigCount.

    std::map<const int64_t, int> alert_stats;



    // Stores the config ID for each sub-config used.

    std::list<std::pair<const int64_t, const int32_t>> annotations;

};



struct UidMapStats {
@@ -142,7 +145,9 @@ public: 
     * If the config is not valid, this config stats will be put into icebox immediately.

     */

    void noteConfigReceived(const ConfigKey& key, int metricsCount, int conditionsCount,

                            int matchersCount, int alertCount, bool isValid);

                            int matchersCount, int alertCount,

                            const std::list<std::pair<const int64_t, const int32_t>>& annotations,

                            bool isValid);

    /**

     * Report a config has been removed.

     */

cmds/statsd/src/metrics/MetricsManager.cpp

+21 −5
Original line number Diff line number Diff line
@@ -33,6 +33,8 @@ 
#include <utils/SystemClock.h>



using android::util::FIELD_COUNT_REPEATED;

using android::util::FIELD_TYPE_INT32;

using android::util::FIELD_TYPE_INT64;

using android::util::FIELD_TYPE_MESSAGE;

using android::util::ProtoOutputStream;
@@ -47,6 +49,9 @@ namespace os { 
namespace statsd {



const int FIELD_ID_METRICS = 1;

const int FIELD_ID_ANNOTATIONS = 7;

const int FIELD_ID_ANNOTATIONS_INT64 = 1;

const int FIELD_ID_ANNOTATIONS_INT32 = 2;



MetricsManager::MetricsManager(const ConfigKey& key, const StatsdConfig& config,

                               const long timeBaseSec, const long currentTimeSec,
@@ -85,6 +90,11 @@ MetricsManager::MetricsManager(const ConfigKey& key, const StatsdConfig& config, 
        }

    }



    // Store the sub-configs used.

    for (const auto& annotation : config.annotation()) {

        mAnnotations.emplace_back(annotation.field_int64(), annotation.field_int32());

    }



    // Guardrail. Reject the config if it's too big.

    if (mAllMetricProducers.size() > StatsdStats::kMaxMetricCountPerConfig ||

        mAllConditionTrackers.size() > StatsdStats::kMaxConditionCountPerConfig ||
@@ -97,11 +107,9 @@ MetricsManager::MetricsManager(const ConfigKey& key, const StatsdConfig& config, 
        mConfigValid = false;

    }

    // no matter whether this config is valid, log it in the stats.

    StatsdStats::getInstance().noteConfigReceived(key, mAllMetricProducers.size(),

                                                  mAllConditionTrackers.size(),

                                                  mAllAtomMatchers.size(),

                                                  mAllAnomalyTrackers.size(),

                                                  mConfigValid);

    StatsdStats::getInstance().noteConfigReceived(

            key, mAllMetricProducers.size(), mAllConditionTrackers.size(), mAllAtomMatchers.size(),

            mAllAnomalyTrackers.size(), mAnnotations, mConfigValid);

}



MetricsManager::~MetricsManager() {
@@ -188,6 +196,14 @@ void MetricsManager::onDumpReport(const uint64_t dumpTimeStampNs, ProtoOutputStr 
            protoOutput->end(token);

        }

    }

    for (const auto& annotation : mAnnotations) {

        uint64_t token = protoOutput->start(FIELD_TYPE_MESSAGE | FIELD_COUNT_REPEATED |

                                            FIELD_ID_ANNOTATIONS);

        protoOutput->write(FIELD_TYPE_INT64 | FIELD_ID_ANNOTATIONS_INT64,

                           (long long)annotation.first);

        protoOutput->write(FIELD_TYPE_INT32 | FIELD_ID_ANNOTATIONS_INT32, annotation.second);

        protoOutput->end(token);

    }

    mLastReportTimeNs = dumpTimeStampNs;

    mLastReportWallClockNs = getWallClockNs();

    VLOG("=========================Metric Reports End==========================");

cmds/statsd/src/metrics/MetricsManager.h

+3 −0
Original line number Diff line number Diff line
@@ -107,6 +107,9 @@ private: 
    // Logs from uids that are not in the list will be ignored to avoid spamming.

    std::set<int32_t> mAllowedLogSources;



    // Contains the annotations passed in with StatsdConfig.

    std::list<std::pair<const int64_t, const int32_t>> mAnnotations;



    // To guard access to mAllowedLogSources

    mutable std::mutex mAllowedLogSourcesMutex;

cmds/statsd/src/stats_log.proto

+11 −0
Original line number Diff line number Diff line
@@ -186,6 +186,12 @@ message ConfigMetricsReport { 
  optional int64 last_report_wall_clock_nanos = 5;



  optional int64 current_report_wall_clock_nanos = 6;



  message Annotation {

      optional int64 field_int64 = 1;

      optional int32 field_int32 = 2;

  }

  repeated Annotation annotation = 7;

}



message ConfigMetricsReportList {
@@ -242,6 +248,11 @@ message StatsdStatsReport { 
        repeated MetricStats metric_stats = 15;

        repeated AlertStats alert_stats = 16;

        repeated MetricStats metric_dimension_in_condition_stats = 17;

        message Annotation {

            optional int64 field_int64 = 1;

            optional int32 field_int32 = 2;

        }

        repeated Annotation annotation = 18;

    }



    repeated ConfigStats config_stats = 3;