Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 3eba6218 authored by Yang Lu's avatar Yang Lu Committed by Yangster
Browse files

Update CountAnomalyTracker to handle slicing. 

Re-added CountAnomalyTracker (under the name DiscreteAnomalyDetector).
It is now able to handle dimensions.

Test: unit test passed.

Change-Id: I133fcf6db99b9e305756f7ee5891c25f0f276348
parent 45a9600e

cmds/statsd/Android.mk

+2 −1
Original line number Diff line number Diff line
@@ -38,7 +38,7 @@ statsd_common_src := \ 
    src/matchers/CombinationLogMatchingTracker.cpp \

    src/matchers/matcher_util.cpp \

    src/matchers/SimpleLogMatchingTracker.cpp \

    src/metrics/CountAnomalyTracker.cpp \

    src/anomaly/DiscreteAnomalyTracker.cpp \

    src/metrics/MetricProducer.cpp \

    src/metrics/EventMetricProducer.cpp \

    src/metrics/CountMetricProducer.cpp \
@@ -150,6 +150,7 @@ LOCAL_CFLAGS += \ 
LOCAL_SRC_FILES := \

    $(statsd_common_src) \

    tests/AnomalyMonitor_test.cpp \

    tests/anomaly/AnomalyTracker_test.cpp \

    tests/ConditionTracker_test.cpp \

    tests/ConfigManager_test.cpp \

    tests/indexed_priority_queue_test.cpp \

cmds/statsd/src/anomaly/AnomalyMonitor.h

+10 −0
Original line number Diff line number Diff line
@@ -55,6 +55,7 @@ struct AnomalyAlarm : public RefBase { 
    };

};



// TODO: Rename this file to AnomalyAlarmMonitor.

/**

 * Manages alarms for Anomaly Detection.

 */
@@ -95,6 +96,15 @@ public: 
    unordered_set<sp<const AnomalyAlarm>, SpHash<AnomalyAlarm>> popSoonerThan(

            uint32_t timestampSec);



    // TODO: Function that uses popSoonerThan to get all alarms that have fired, and then

    // iterates over all DurationAnomalyTracker, looking for those alarms. When they're found,

    // have them declareAnomaly on those alarms. This means that DurationAnomalyTracker

    // must be thread-safe (since this is being called on a different thread). There is no

    // worry about missing the alarms (due to them being cancelled after this function being called)

    // because DurationAnomalyTracker guarantees that it checks for anaomlies when it cancels

    // alarms anyway.

    // void declareAnomalies(uint32_t timestampSec);



    /**

     * Returns the projected alarm timestamp that is registered with

     * StatsCompanionService. This may not be equal to the soonest alarm,

cmds/statsd/src/metrics/CountAnomalyTracker.cpp→cmds/statsd/src/anomaly/DiscreteAnomalyTracker.cpp

+159 −0
Original line number Diff line number Diff line
@@ -17,7 +17,7 @@ 
#define DEBUG true  // STOPSHIP if true

#include "Log.h"



#include "CountAnomalyTracker.h"

#include "DiscreteAnomalyTracker.h"



#include <time.h>
@@ -25,89 +25,119 @@ namespace android { 
namespace os {

namespace statsd {



CountAnomalyTracker::CountAnomalyTracker(const Alert& alert)

    : mAlert(alert),

      mNumPastBuckets(alert.number_of_buckets() > 0 ? alert.number_of_buckets() - 1 : 0),

      mPastBuckets(mNumPastBuckets > 0 ? (new int[mNumPastBuckets]) : nullptr) {



    VLOG("CountAnomalyTracker() called");

    if (alert.number_of_buckets() < 1) {

        ALOGE("Cannot create CountAnomalyTracker with %d buckets", alert.number_of_buckets());

DiscreteAnomalyTracker::DiscreteAnomalyTracker(const Alert& alert) : mAlert(alert) {

    VLOG("DiscreteAnomalyTracker() called");

    if (mAlert.number_of_buckets() <= 0) {

        ALOGE("Cannot create DiscreteAnomalyTracker with %lld buckets",

              (long long)mAlert.number_of_buckets());

        return;

    }

    mPastBuckets.resize(mAlert.number_of_buckets());

    reset(); // initialization

}



CountAnomalyTracker::~CountAnomalyTracker() {

    VLOG("~CountAnomalyTracker() called");

DiscreteAnomalyTracker::~DiscreteAnomalyTracker() {

    VLOG("~DiscreteAnomalyTracker() called");

}



void CountAnomalyTracker::addPastBucket(int pastBucketCount,

                                        time_t numberOfBucketsAgo) {

    VLOG("addPastBucket() called.");

    if (numberOfBucketsAgo < 1) {

        ALOGE("Cannot add a past bucket %ld units in past", numberOfBucketsAgo);

        return;

void DiscreteAnomalyTracker::reset() {

    VLOG("reset() called.");

    mPastBuckets.clear();

    mPastBuckets.resize(mAlert.number_of_buckets());

    mSumOverPastBuckets.clear();

    mCurrentBucketIndex = -1;

    mLastAlarmAtBucketIndex = -1;

    mAnomalyDeclared = 0;

}



size_t DiscreteAnomalyTracker::index(int64_t bucketNum) {

    return bucketNum % mAlert.number_of_buckets();

}

    // If past bucket was ancient, just empty out all past info.

    // This always applies if mNumPastBuckets == 0 (i.e. store no past buckets).

    if (numberOfBucketsAgo > (time_t) mNumPastBuckets) {

        reset();



void DiscreteAnomalyTracker::addOrUpdateBucket(std::shared_ptr<const DimToValMap> BucketValues,

                                               int64_t bucketIndex) {

    VLOG("addPastBucket() called.");

    if (bucketIndex <= mCurrentBucketIndex - mAlert.number_of_buckets()) {

        ALOGE("Cannot add a past bucket %lld units in past", (long long)bucketIndex);

        return;

    }



    // Empty out old mPastBuckets[i] values and update mSumPastCounters.

    for (size_t i = mOldestBucketIndex;

                        i < mOldestBucketIndex + numberOfBucketsAgo; i++) {

        mSumPastCounters -= mPastBuckets[index(i)];

        mPastBuckets[index(i)] = 0;

    // Empty out old mPastBuckets[i] values and update mSumOverPastBuckets.

    if (bucketIndex - mCurrentBucketIndex >= mAlert.number_of_buckets()) {

        mPastBuckets.clear();

        mPastBuckets.resize(mAlert.number_of_buckets());

        mSumOverPastBuckets.clear();

    } else {

        for (int64_t i = std::max(

                     0LL, (long long)(mCurrentBucketIndex - mAlert.number_of_buckets() + 1));

             i < bucketIndex - mAlert.number_of_buckets(); i++) {

            const int idx = index(i);

            subtractBucketFromSum(mPastBuckets[idx]);

            mPastBuckets[idx] = nullptr;  // release (but not clear) the old bucket.

        }

    }

    subtractBucketFromSum(mPastBuckets[index(bucketIndex)]);

    mPastBuckets[index(bucketIndex)] = nullptr;  // release (but not clear) the old bucket.



    // Replace the oldest bucket with the new bucket we are adding.

    mPastBuckets[mOldestBucketIndex] = pastBucketCount;

    mSumPastCounters += pastBucketCount;

    mPastBuckets[index(bucketIndex)] = BucketValues;

    addBucketToSum(BucketValues);



    // Advance the oldest bucket index by numberOfBucketsAgo units.

    mOldestBucketIndex = index(mOldestBucketIndex + numberOfBucketsAgo);



    // TODO: Once dimensions are added to mSumPastCounters:

    // iterate through mSumPastCounters and remove any entries that are 0.

    mCurrentBucketIndex = std::max(mCurrentBucketIndex, bucketIndex);

}



void CountAnomalyTracker::reset() {

    VLOG("reset() called.");

    for (size_t i = 0; i < mNumPastBuckets; i++) {

        mPastBuckets[i] = 0;

void DiscreteAnomalyTracker::subtractBucketFromSum(const shared_ptr<const DimToValMap>& bucket) {

    if (bucket == nullptr) {

        return;

    }

    // For each dimension present in the bucket, subtract its value from its corresponding sum.

    for (const auto& keyValuePair : *bucket) {

        auto itr = mSumOverPastBuckets.find(keyValuePair.first);

        if (itr == mSumOverPastBuckets.end()) {

            continue;

        }

        itr->second -= keyValuePair.second;

        // TODO: No need to look up the object twice like this. Use a var.

        if (itr->second == 0) {

            mSumOverPastBuckets.erase(itr);

        }

    }

    mSumPastCounters = 0;

    mOldestBucketIndex = 0;

}



void CountAnomalyTracker::checkAnomaly(int currentCount) {

    // Skip the check if in refractory period.

    if (time(nullptr) < mRefractoryPeriodEndsSec) {

        VLOG("Skipping anomaly check since within refractory period");

void DiscreteAnomalyTracker::addBucketToSum(const shared_ptr<const DimToValMap>& bucket) {

    if (bucket == nullptr) {

        return;

    }

    // For each dimension present in the bucket, add its value to its corresponding sum.

    for (const auto& keyValuePair : *bucket) {

        mSumOverPastBuckets[keyValuePair.first] += keyValuePair.second;

    }

}



    // TODO: Remove these extremely verbose debugging log.

    VLOG("Checking whether %d + %d > %lld",

         mSumPastCounters, currentCount, mAlert.trigger_if_sum_gt());

bool DiscreteAnomalyTracker::detectAnomaly() {

    for (auto itr = mSumOverPastBuckets.begin(); itr != mSumOverPastBuckets.end(); itr++) {

        if (mAlert.has_trigger_if_sum_gt() && itr->second > mAlert.trigger_if_sum_gt()) {

            return true;

        }

    }

    return false;

}



    // Note that this works even if mNumPastBuckets < 1 (since then

    // mSumPastCounters = 0 so the comparison is based only on currentCount).

    if (mAlert.has_trigger_if_sum_gt() &&

            mSumPastCounters + currentCount > mAlert.trigger_if_sum_gt()) {

void DiscreteAnomalyTracker::declareAndDeclareAnomaly() {

    if (detectAnomaly()) {

        declareAnomaly();

    }

}



void CountAnomalyTracker::declareAnomaly() {

void DiscreteAnomalyTracker::declareAnomaly() {

    if (mLastAlarmAtBucketIndex >= 0 && mCurrentBucketIndex - mLastAlarmAtBucketIndex <=

                                        (long long)mAlert.refractory_period_in_buckets()) {

        VLOG("Skipping anomaly check since within refractory period");

        return;

    }

    mAnomalyDeclared++;

    // TODO(guardrail): Consider guarding against too short refractory periods.

    time_t currTime = time(nullptr);

    mRefractoryPeriodEndsSec = currTime + mAlert.refractory_period_secs();



    // TODO: If we had access to the bucket_size_millis, consider calling reset()

    // if (mAlert.refractory_period_secs() > mNumPastBuckets * bucket_size_millis * 1000).

    mLastAlarmAtBucketIndex = mCurrentBucketIndex;



    if (mAlert.has_incidentd_details()) {

        const Alert_IncidentdDetails& incident = mAlert.incidentd_details();

cmds/statsd/src/metrics/CountAnomalyTracker.h→cmds/statsd/src/anomaly/DiscreteAnomalyTracker.h

+91 −0
Original line number Diff line number Diff line
@@ -14,10 +14,11 @@ 
 * limitations under the License.

 */



#ifndef COUNT_ANOMALY_TRACKER_H

#define COUNT_ANOMALY_TRACKER_H

#pragma once



#include <gtest/gtest_prod.h>

#include "frameworks/base/cmds/statsd/src/statsd_config.pb.h" // Alert

#include "stats_util.h" // HashableDimensionKey and DimToValMap



#include <memory> // unique_ptr

#include <stdlib.h>
@@ -26,62 +27,65 @@ namespace android { 
namespace os {

namespace statsd {



// TODO: Can probably be used for Count, Value, and Gauge. If so, rename to ValueAnomalyTracker.

// (caveat: currently, the value cannot be negative. Probably fine for P.)

class CountAnomalyTracker {

using std::unordered_map;

using std::shared_ptr;



// This anomaly track assmues that all values are non-negative.

class DiscreteAnomalyTracker {

 public:

    CountAnomalyTracker(const Alert& alert);

    DiscreteAnomalyTracker(const Alert& alert);



    virtual ~DiscreteAnomalyTracker();



    virtual ~CountAnomalyTracker();

    // Adds a new bucket or updates an existing bucket.

    // Bucket index starts from 0.

    void addOrUpdateBucket(std::shared_ptr<const DimToValMap> BucketValues, int64_t bucketIndex);



    // Returns true if detected anomaly for the existing buckets on one or more dimension keys.

    bool detectAnomaly();



    // Adds a new past bucket, holding pastBucketCount, and then advances the

    // present by numberOfBucketsAgo buckets (filling any intervening buckets

    // with 0s).

    // Thus, the newly added bucket (which holds pastBucketCount) is stored

    // numberOfBucketsAgo buckets ago.

    void addPastBucket(int pastBucketCount, time_t numberOfBucketsAgo);

    // Informs incidentd about the detected alert.

    void declareAnomaly();



    // Informs the anomaly tracker of the current bucket's count, so that it can

    // determine whether an anomaly has occurred. This value is not stored.

    void checkAnomaly(int currentCount);

    // Detects the alert and informs the incidentd when applicable.

    void declareAndDeclareAnomaly();



private:

    // statsd_config.proto Alert message that defines this tracker.

    const Alert mAlert;



    // Number of past buckets. One less than the total number of buckets needed

    // for the anomaly detection (since the current bucket is not in the past).

    const size_t mNumPastBuckets;

    // The exisiting bucket list.

    std::vector<shared_ptr<const DimToValMap>> mPastBuckets;



    // Count values for each of the past mNumPastBuckets buckets.

    // TODO: Add dimensions. This parallels the type of CountMetricProducer.mCounter.

    std::unique_ptr<int[]> mPastBuckets;

    // Sum over all existing buckets cached in mPastBuckets.

    DimToValMap mSumOverPastBuckets;



    // Sum over all of mPastBuckets (cached).

    // TODO: Add dimensions. This parallels the type of CountMetricProducer.mCounter.

    //       At that point, mSumPastCounters must never contain entries of 0.

    int mSumPastCounters;

    // Current bucket index of the current anomaly detection window. Bucket index starts from 0.

    int64_t mCurrentBucketIndex = -1;



    // Index of the oldest bucket (i.e. the next bucket to be overwritten).

    size_t mOldestBucketIndex = 0;

    // The bucket index when the last anomaly was declared.

    int64_t mLastAlarmAtBucketIndex = -1;



    // Timestamp that the refractory period (if this anomaly was declared) ends, in seconds.

    // If an anomaly was never declared, set to 0.

    time_t mRefractoryPeriodEndsSec = 0;

    // The total number of declared anomalies.

    int64_t mAnomalyDeclared = 0;



    void declareAnomaly();

    // Add the information in the given bucket to mSumOverPastBuckets.

    void addBucketToSum(const shared_ptr<const DimToValMap>& bucket);



    // Subtract the information in the given bucket from mSumOverPastBuckets

    // and remove any items with value 0.

    void subtractBucketFromSum(const shared_ptr<const DimToValMap>& bucket);



    // Calculates the corresponding index within the circular array.

    size_t index(size_t unsafeIndex) {

        return unsafeIndex % mNumPastBuckets;

    }

    // Calculates the corresponding bucket index within the circular array.

    size_t index(int64_t bucketNum);



    // Resets all data. For use when all the data gets stale.

    void reset();



    FRIEND_TEST(AnomalyTrackerTest, TestConsecutiveBuckets);

    FRIEND_TEST(AnomalyTrackerTest, TestSparseBuckets);

};



}  // namespace statsd

}  // namespace os

}  // namespace android
 
#endif  // COUNT_ANOMALY_TRACKER_H

cmds/statsd/src/config/ConfigManager.cpp

+6 −0
Original line number Diff line number Diff line
@@ -165,6 +165,12 @@ static StatsdConfig build_fake_config() { 
    KeyMatcher* keyMatcher = metric->add_dimension();

    keyMatcher->set_key(UID_PROCESS_STATE_UID_KEY);



    // Anomaly threshold for background count.

    alert = metric->add_alerts();

    alert->set_number_of_buckets(4);

    alert->set_trigger_if_sum_gt(30);

    alert->set_refractory_period_secs(20);



    // Count process state changes, slice by uid, while SCREEN_IS_OFF

    metric = config.add_count_metric();

    metric->set_metric_id(3);