Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 2087716f authored by Yangster-mac's avatar Yangster-mac
Browse files

1/ Support nested message and repeated fields in statsd. 

2/ Filter gauge fields by FieldMatcher.
3/ Wire up wakelock attribution chain.
4/ e2e test: wakelock duration metric with aggregated predicate dimensions.
5/ e2e test: count metric with multiple metric condition links for 2 predicates and 1 non-sliced predicate.

Test: statsd unit test passed.

Change-Id: I89db31cb068184a54e0a892fad710966d3127bc9
parent 28bc987f

cmds/statsd/Android.mk

+9 −2
Original line number Diff line number Diff line
@@ -20,6 +20,9 @@ statsd_common_src := \ 
    src/stats_log.proto \

    src/statsd_config.proto \

    src/atoms.proto \

    src/field_util.cpp \

    src/stats_log_util.cpp \

    src/dimension.cpp \

    src/anomaly/AnomalyMonitor.cpp \

    src/anomaly/AnomalyTracker.cpp \

    src/anomaly/DurationAnomalyTracker.cpp \
@@ -163,6 +166,7 @@ LOCAL_SRC_FILES := \ 
    tests/indexed_priority_queue_test.cpp \

    tests/LogEntryMatcher_test.cpp \

    tests/LogReader_test.cpp \

    tests/LogEvent_test.cpp \

    tests/MetricsManager_test.cpp \

    tests/StatsLogProcessor_test.cpp \

    tests/UidMap_test.cpp \
@@ -176,7 +180,10 @@ LOCAL_SRC_FILES := \ 
    tests/metrics/ValueMetricProducer_test.cpp \

    tests/metrics/GaugeMetricProducer_test.cpp \

    tests/guardrail/StatsdStats_test.cpp \

    tests/metrics/metrics_test_helper.cpp

    tests/metrics/metrics_test_helper.cpp \

    tests/statsd_test_util.cpp \

    tests/e2e/WakelockDuration_e2e_test.cpp \

    tests/e2e/MetricConditionLink_e2e_test.cpp



LOCAL_STATIC_LIBRARIES := \

    $(statsd_common_static_libraries) \

cmds/statsd/src/HashableDimensionKey.cpp

+78 −66
Original line number Diff line number Diff line
@@ -13,92 +13,104 @@ 
 * See the License for the specific language governing permissions and

 * limitations under the License.

 */



#include "HashableDimensionKey.h"

#include "dimension.h"



namespace android {

namespace os {

namespace statsd {



using std::string;

android::hash_t hashDimensionsValue(const DimensionsValue& value) {

    android::hash_t hash = 0;

    hash = android::JenkinsHashMix(hash, android::hash_type(value.field()));



string HashableDimensionKey::toString() const {

    string flattened;

    for (const auto& pair : mKeyValuePairs) {

        flattened += std::to_string(pair.key());

        flattened += ":";

        switch (pair.value_case()) {

            case KeyValuePair::ValueCase::kValueStr:

                flattened += pair.value_str();

    hash = android::JenkinsHashMix(hash, android::hash_type((int)value.value_case()));

    switch (value.value_case()) {

        case DimensionsValue::ValueCase::kValueStr:

            hash = android::JenkinsHashMix(

                    hash,

                    static_cast<uint32_t>(std::hash<std::string>()(value.value_str())));

            break;

            case KeyValuePair::ValueCase::kValueInt:

                flattened += std::to_string(pair.value_int());

        case DimensionsValue::ValueCase::kValueInt:

            hash = android::JenkinsHashMix(hash, android::hash_type(value.value_int()));

            break;

            case KeyValuePair::ValueCase::kValueLong:

                flattened += std::to_string(pair.value_long());

        case DimensionsValue::ValueCase::kValueLong:

            hash = android::JenkinsHashMix(

                    hash, android::hash_type(static_cast<int64_t>(value.value_long())));

            break;

            case KeyValuePair::ValueCase::kValueBool:

                flattened += std::to_string(pair.value_bool());

        case DimensionsValue::ValueCase::kValueBool:

            hash = android::JenkinsHashMix(hash, android::hash_type(value.value_bool()));

            break;

            case KeyValuePair::ValueCase::kValueFloat:

                flattened += std::to_string(pair.value_float());

        case DimensionsValue::ValueCase::kValueFloat: {

            float floatVal = value.value_float();

            hash = android::JenkinsHashMixBytes(hash, (uint8_t*)&floatVal, sizeof(float));

            break;

            default:

        }

        case DimensionsValue::ValueCase::kValueTuple: {

            hash = android::JenkinsHashMix(hash, android::hash_type(

                value.value_tuple().dimensions_value_size()));

            for (int i = 0; i < value.value_tuple().dimensions_value_size(); ++i) {

                hash = android::JenkinsHashMix(

                    hash,

                    hashDimensionsValue(value.value_tuple().dimensions_value(i)));

            }

            break;

        }

        flattened += "|";

        case DimensionsValue::ValueCase::VALUE_NOT_SET:

            break;

    }

    return flattened;

    return JenkinsHashWhiten(hash);

}



bool HashableDimensionKey::operator==(const HashableDimensionKey& that) const {

    const auto& keyValue2 = that.getKeyValuePairs();

    if (mKeyValuePairs.size() != keyValue2.size()) {

        return false;

    }

using std::string;



    for (size_t i = 0; i < keyValue2.size(); i++) {

        const auto& kv1 = mKeyValuePairs[i];

        const auto& kv2 = keyValue2[i];

        if (kv1.key() != kv2.key()) {

            return false;

        }



        if (kv1.value_case() != kv2.value_case()) {

            return false;

string HashableDimensionKey::toString() const {

    string flattened;

    DimensionsValueToString(getDimensionsValue(), &flattened);

    return flattened;

}



        switch (kv1.value_case()) {

            case KeyValuePair::ValueCase::kValueStr:

                if (kv1.value_str() != kv2.value_str()) {

                    return false;

                }

                break;

            case KeyValuePair::ValueCase::kValueInt:

                if (kv1.value_int() != kv2.value_int()) {

bool compareDimensionsValue(const DimensionsValue& s1, const DimensionsValue& s2) {

    if (s1.field() != s2.field()) {

        return false;

    }

                break;

            case KeyValuePair::ValueCase::kValueLong:

                if (kv1.value_long() != kv2.value_long()) {

    if (s1.value_case() != s1.value_case()) {

        return false;

    }

                break;

            case KeyValuePair::ValueCase::kValueBool:

                if (kv1.value_bool() != kv2.value_bool()) {

    switch (s1.value_case()) {

        case DimensionsValue::ValueCase::kValueStr:

            return (s1.value_str() == s2.value_str());

        case DimensionsValue::ValueCase::kValueInt:

            return s1.value_int() == s2.value_int();

        case DimensionsValue::ValueCase::kValueLong:

            return s1.value_long() == s2.value_long();

        case DimensionsValue::ValueCase::kValueBool:

            return s1.value_bool() == s2.value_bool();

        case DimensionsValue::ValueCase::kValueFloat:

            return s1.value_float() == s2.value_float();

        case DimensionsValue::ValueCase::kValueTuple:

            {

                if (s1.value_tuple().dimensions_value_size() !=

                        s2.value_tuple().dimensions_value_size()) {

                    return false;

                }

                break;

            case KeyValuePair::ValueCase::kValueFloat: {

                if (kv1.value_float() != kv2.value_float()) {

                    return false;

                bool allMatched = true;

                for (int i = 0; allMatched && i < s1.value_tuple().dimensions_value_size(); ++i) {

                    allMatched &= compareDimensionsValue(s1.value_tuple().dimensions_value(i),

                                                    s2.value_tuple().dimensions_value(i));

                }

                break;

                return allMatched;

            }

            case KeyValuePair::ValueCase::VALUE_NOT_SET:

                break;

        case DimensionsValue::ValueCase::VALUE_NOT_SET:

        default:

            return true;

    }

}

    return true;



bool HashableDimensionKey::operator==(const HashableDimensionKey& that) const {

    return compareDimensionsValue(getDimensionsValue(), that.getDimensionsValue());

};



bool HashableDimensionKey::operator<(const HashableDimensionKey& that) const {

cmds/statsd/src/HashableDimensionKey.h

+9 −38
Original line number Diff line number Diff line
@@ -25,20 +25,20 @@ namespace statsd { 


class HashableDimensionKey {

public:

    explicit HashableDimensionKey(const std::vector<KeyValuePair>& keyValuePairs)

        : mKeyValuePairs(keyValuePairs){};

    explicit HashableDimensionKey(const DimensionsValue& dimensionsValue)

        : mDimensionsValue(dimensionsValue){};



    HashableDimensionKey(){};



    HashableDimensionKey(const HashableDimensionKey& that)

        : mKeyValuePairs(that.getKeyValuePairs()){};

        : mDimensionsValue(that.getDimensionsValue()){};



    HashableDimensionKey& operator=(const HashableDimensionKey& from) = default;



    std::string toString() const;



    inline const std::vector<KeyValuePair>& getKeyValuePairs() const {

        return mKeyValuePairs;

    inline const DimensionsValue& getDimensionsValue() const {

        return mDimensionsValue;

    }



    bool operator==(const HashableDimensionKey& that) const;
@@ -50,9 +50,11 @@ public: 
    }



private:

    std::vector<KeyValuePair> mKeyValuePairs;

    DimensionsValue mDimensionsValue;

};



android::hash_t hashDimensionsValue(const DimensionsValue& value);



}  // namespace statsd

}  // namespace os

}  // namespace android
@@ -60,42 +62,11 @@ private: 
namespace std {



using android::os::statsd::HashableDimensionKey;

using android::os::statsd::KeyValuePair;



template <>

struct hash<HashableDimensionKey> {

    std::size_t operator()(const HashableDimensionKey& key) const {

        android::hash_t hash = 0;

        for (const auto& pair : key.getKeyValuePairs()) {

            hash = android::JenkinsHashMix(hash, android::hash_type(pair.key()));

            hash = android::JenkinsHashMix(

                    hash, android::hash_type(static_cast<int32_t>(pair.value_case())));

            switch (pair.value_case()) {

                case KeyValuePair::ValueCase::kValueStr:

                    hash = android::JenkinsHashMix(

                            hash,

                            static_cast<uint32_t>(std::hash<std::string>()(pair.value_str())));

                    break;

                case KeyValuePair::ValueCase::kValueInt:

                    hash = android::JenkinsHashMix(hash, android::hash_type(pair.value_int()));

                    break;

                case KeyValuePair::ValueCase::kValueLong:

                    hash = android::JenkinsHashMix(

                            hash, android::hash_type(static_cast<int64_t>(pair.value_long())));

                    break;

                case KeyValuePair::ValueCase::kValueBool:

                    hash = android::JenkinsHashMix(hash, android::hash_type(pair.value_bool()));

                    break;

                case KeyValuePair::ValueCase::kValueFloat: {

                    float floatVal = pair.value_float();

                    hash = android::JenkinsHashMixBytes(hash, (uint8_t*)&floatVal, sizeof(float));

                    break;

                }

                case KeyValuePair::ValueCase::VALUE_NOT_SET:

                    break;

            }

        }

        return hash;

        return hashDimensionsValue(key.getDimensionsValue());

    }

};

cmds/statsd/src/StatsLogProcessor.cpp

+15 −4
Original line number Diff line number Diff line
@@ -63,11 +63,12 @@ const int FIELD_ID_UID_MAP = 2; 


StatsLogProcessor::StatsLogProcessor(const sp<UidMap>& uidMap,

                                     const sp<AnomalyMonitor>& anomalyMonitor,

                                     const long timeBaseSec,

                                     const std::function<void(const ConfigKey&)>& sendBroadcast)

    : mUidMap(uidMap),

      mAnomalyMonitor(anomalyMonitor),

      mSendBroadcast(sendBroadcast),

      mTimeBaseSec(time(nullptr)) {

      mTimeBaseSec(timeBaseSec) {

    // On each initialization of StatsLogProcessor, check stats-data directory to see if there is

    // any left over data to be read.

    StorageManager::sendBroadcast(STATS_DATA_DIR, mSendBroadcast);
@@ -97,7 +98,6 @@ void StatsLogProcessor::OnLogEvent(const LogEvent& msg) { 
        pair.second->onLogEvent(msg);

        flushIfNecessary(msg.GetTimestampNs(), pair.first, *(pair.second));

    }



    // Hard-coded logic to update the isolated uid's in the uid-map.

    // The field numbers need to be currently updated by hand with atoms.proto

    if (msg.GetTagId() == android::util::ISOLATED_UID_CHANGED) {
@@ -117,9 +117,7 @@ void StatsLogProcessor::OnLogEvent(const LogEvent& msg) { 


void StatsLogProcessor::OnConfigUpdated(const ConfigKey& key, const StatsdConfig& config) {

    ALOGD("Updated configuration for key %s", key.ToString().c_str());



    sp<MetricsManager> newMetricsManager = new MetricsManager(key, config, mTimeBaseSec, mUidMap);



    auto it = mMetricsManagers.find(key);

    if (it == mMetricsManagers.end() && mMetricsManagers.size() > StatsdStats::kMaxConfigCount) {

        ALOGE("Can't accept more configs!");
@@ -152,6 +150,19 @@ size_t StatsLogProcessor::GetMetricsSize(const ConfigKey& key) const { 
    return it->second->byteSize();

}



void StatsLogProcessor::onDumpReport(const ConfigKey& key, const uint64_t& dumpTimeStampNs, ConfigMetricsReportList* report) {

    auto it = mMetricsManagers.find(key);

    if (it == mMetricsManagers.end()) {

        ALOGW("Config source %s does not exist", key.ToString().c_str());

        return;

    }

    report->mutable_config_key()->set_uid(key.GetUid());

    report->mutable_config_key()->set_name(key.GetName());

    ConfigMetricsReport* configMetricsReport = report->add_reports();

    it->second->onDumpReport(dumpTimeStampNs, configMetricsReport);

    // TODO: dump uid mapping.

}



void StatsLogProcessor::onDumpReport(const ConfigKey& key, vector<uint8_t>* outData) {

    auto it = mMetricsManagers.find(key);

    if (it == mMetricsManagers.end()) {

cmds/statsd/src/StatsLogProcessor.h

+9 −4
Original line number Diff line number Diff line
@@ -13,9 +13,10 @@ 
 * See the License for the specific language governing permissions and

 * limitations under the License.

 */

#ifndef STATS_LOG_PROCESSOR_H

#define STATS_LOG_PROCESSOR_H



#pragma once



#include <gtest/gtest_prod.h>

#include "config/ConfigListener.h"

#include "logd/LogReader.h"

#include "metrics/MetricsManager.h"
@@ -33,6 +34,7 @@ namespace statsd { 
class StatsLogProcessor : public ConfigListener {

public:

    StatsLogProcessor(const sp<UidMap>& uidMap, const sp<AnomalyMonitor>& anomalyMonitor,

                      const long timeBaseSec,

                      const std::function<void(const ConfigKey&)>& sendBroadcast);

    virtual ~StatsLogProcessor();
@@ -44,6 +46,7 @@ public: 
    size_t GetMetricsSize(const ConfigKey& key) const;



    void onDumpReport(const ConfigKey& key, vector<uint8_t>* outData);

    void onDumpReport(const ConfigKey& key, const uint64_t& dumpTimeStampNs, ConfigMetricsReportList* report);



    /* Tells MetricsManager that the alarms in anomalySet have fired. Modifies anomalySet. */

    void onAnomalyAlarmFired(
@@ -81,10 +84,12 @@ private: 
    FRIEND_TEST(StatsLogProcessorTest, TestRateLimitByteSize);

    FRIEND_TEST(StatsLogProcessorTest, TestRateLimitBroadcast);

    FRIEND_TEST(StatsLogProcessorTest, TestDropWhenByteSizeTooLarge);

    FRIEND_TEST(StatsLogProcessorTest, TestDropWhenByteSizeTooLarge);

    FRIEND_TEST(WakelockDurationE2eTest, TestAggregatedPredicateDimensions);

    FRIEND_TEST(MetricConditionLinkE2eTest, TestMultiplePredicatesAndLinks);



};



}  // namespace statsd

}  // namespace os

}  // namespace android
 


#endif  // STATS_LOG_PROCESSOR_H