Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 3eba6218 authored by Yang Lu's avatar Yang Lu Committed by Yangster
Browse files

Update CountAnomalyTracker to handle slicing.

Re-added CountAnomalyTracker (under the name DiscreteAnomalyDetector).
It is now able to handle dimensions.

Test: unit test passed.

Change-Id: I133fcf6db99b9e305756f7ee5891c25f0f276348
parent 45a9600e
Loading
Loading
Loading
Loading
+2 −1
Original line number Diff line number Diff line
@@ -38,7 +38,7 @@ statsd_common_src := \
    src/matchers/CombinationLogMatchingTracker.cpp \
    src/matchers/matcher_util.cpp \
    src/matchers/SimpleLogMatchingTracker.cpp \
    src/metrics/CountAnomalyTracker.cpp \
    src/anomaly/DiscreteAnomalyTracker.cpp \
    src/metrics/MetricProducer.cpp \
    src/metrics/EventMetricProducer.cpp \
    src/metrics/CountMetricProducer.cpp \
@@ -150,6 +150,7 @@ LOCAL_CFLAGS += \
LOCAL_SRC_FILES := \
    $(statsd_common_src) \
    tests/AnomalyMonitor_test.cpp \
    tests/anomaly/AnomalyTracker_test.cpp \
    tests/ConditionTracker_test.cpp \
    tests/ConfigManager_test.cpp \
    tests/indexed_priority_queue_test.cpp \
+10 −0
Original line number Diff line number Diff line
@@ -55,6 +55,7 @@ struct AnomalyAlarm : public RefBase {
    };
};

// TODO: Rename this file to AnomalyAlarmMonitor.
/**
 * Manages alarms for Anomaly Detection.
 */
@@ -95,6 +96,15 @@ public:
    unordered_set<sp<const AnomalyAlarm>, SpHash<AnomalyAlarm>> popSoonerThan(
            uint32_t timestampSec);

    // TODO: Function that uses popSoonerThan to get all alarms that have fired, and then
    // iterates over all DurationAnomalyTracker, looking for those alarms. When they're found,
    // have them declareAnomaly on those alarms. This means that DurationAnomalyTracker
    // must be thread-safe (since this is being called on a different thread). There is no
    // worry about missing the alarms (due to them being cancelled after this function being called)
    // because DurationAnomalyTracker guarantees that it checks for anaomlies when it cancels
    // alarms anyway.
    // void declareAnomalies(uint32_t timestampSec);

    /**
     * Returns the projected alarm timestamp that is registered with
     * StatsCompanionService. This may not be equal to the soonest alarm,
+159 −0
Original line number Diff line number Diff line
@@ -17,7 +17,7 @@
#define DEBUG true  // STOPSHIP if true
#include "Log.h"

#include "CountAnomalyTracker.h"
#include "DiscreteAnomalyTracker.h"

#include <time.h>

@@ -25,89 +25,119 @@ namespace android {
namespace os {
namespace statsd {

CountAnomalyTracker::CountAnomalyTracker(const Alert& alert)
    : mAlert(alert),
      mNumPastBuckets(alert.number_of_buckets() > 0 ? alert.number_of_buckets() - 1 : 0),
      mPastBuckets(mNumPastBuckets > 0 ? (new int[mNumPastBuckets]) : nullptr) {

    VLOG("CountAnomalyTracker() called");
    if (alert.number_of_buckets() < 1) {
        ALOGE("Cannot create CountAnomalyTracker with %d buckets", alert.number_of_buckets());
DiscreteAnomalyTracker::DiscreteAnomalyTracker(const Alert& alert) : mAlert(alert) {
    VLOG("DiscreteAnomalyTracker() called");
    if (mAlert.number_of_buckets() <= 0) {
        ALOGE("Cannot create DiscreteAnomalyTracker with %lld buckets",
              (long long)mAlert.number_of_buckets());
        return;
    }
    mPastBuckets.resize(mAlert.number_of_buckets());
    reset(); // initialization
}

CountAnomalyTracker::~CountAnomalyTracker() {
    VLOG("~CountAnomalyTracker() called");
DiscreteAnomalyTracker::~DiscreteAnomalyTracker() {
    VLOG("~DiscreteAnomalyTracker() called");
}

void CountAnomalyTracker::addPastBucket(int pastBucketCount,
                                        time_t numberOfBucketsAgo) {
    VLOG("addPastBucket() called.");
    if (numberOfBucketsAgo < 1) {
        ALOGE("Cannot add a past bucket %ld units in past", numberOfBucketsAgo);
        return;
void DiscreteAnomalyTracker::reset() {
    VLOG("reset() called.");
    mPastBuckets.clear();
    mPastBuckets.resize(mAlert.number_of_buckets());
    mSumOverPastBuckets.clear();
    mCurrentBucketIndex = -1;
    mLastAlarmAtBucketIndex = -1;
    mAnomalyDeclared = 0;
}

size_t DiscreteAnomalyTracker::index(int64_t bucketNum) {
    return bucketNum % mAlert.number_of_buckets();
}
    // If past bucket was ancient, just empty out all past info.
    // This always applies if mNumPastBuckets == 0 (i.e. store no past buckets).
    if (numberOfBucketsAgo > (time_t) mNumPastBuckets) {
        reset();

void DiscreteAnomalyTracker::addOrUpdateBucket(std::shared_ptr<const DimToValMap> BucketValues,
                                               int64_t bucketIndex) {
    VLOG("addPastBucket() called.");
    if (bucketIndex <= mCurrentBucketIndex - mAlert.number_of_buckets()) {
        ALOGE("Cannot add a past bucket %lld units in past", (long long)bucketIndex);
        return;
    }

    // Empty out old mPastBuckets[i] values and update mSumPastCounters.
    for (size_t i = mOldestBucketIndex;
                        i < mOldestBucketIndex + numberOfBucketsAgo; i++) {
        mSumPastCounters -= mPastBuckets[index(i)];
        mPastBuckets[index(i)] = 0;
    // Empty out old mPastBuckets[i] values and update mSumOverPastBuckets.
    if (bucketIndex - mCurrentBucketIndex >= mAlert.number_of_buckets()) {
        mPastBuckets.clear();
        mPastBuckets.resize(mAlert.number_of_buckets());
        mSumOverPastBuckets.clear();
    } else {
        for (int64_t i = std::max(
                     0LL, (long long)(mCurrentBucketIndex - mAlert.number_of_buckets() + 1));
             i < bucketIndex - mAlert.number_of_buckets(); i++) {
            const int idx = index(i);
            subtractBucketFromSum(mPastBuckets[idx]);
            mPastBuckets[idx] = nullptr;  // release (but not clear) the old bucket.
        }
    }
    subtractBucketFromSum(mPastBuckets[index(bucketIndex)]);
    mPastBuckets[index(bucketIndex)] = nullptr;  // release (but not clear) the old bucket.

    // Replace the oldest bucket with the new bucket we are adding.
    mPastBuckets[mOldestBucketIndex] = pastBucketCount;
    mSumPastCounters += pastBucketCount;
    mPastBuckets[index(bucketIndex)] = BucketValues;
    addBucketToSum(BucketValues);

    // Advance the oldest bucket index by numberOfBucketsAgo units.
    mOldestBucketIndex = index(mOldestBucketIndex + numberOfBucketsAgo);

    // TODO: Once dimensions are added to mSumPastCounters:
    // iterate through mSumPastCounters and remove any entries that are 0.
    mCurrentBucketIndex = std::max(mCurrentBucketIndex, bucketIndex);
}

void CountAnomalyTracker::reset() {
    VLOG("reset() called.");
    for (size_t i = 0; i < mNumPastBuckets; i++) {
        mPastBuckets[i] = 0;
void DiscreteAnomalyTracker::subtractBucketFromSum(const shared_ptr<const DimToValMap>& bucket) {
    if (bucket == nullptr) {
        return;
    }
    // For each dimension present in the bucket, subtract its value from its corresponding sum.
    for (const auto& keyValuePair : *bucket) {
        auto itr = mSumOverPastBuckets.find(keyValuePair.first);
        if (itr == mSumOverPastBuckets.end()) {
            continue;
        }
        itr->second -= keyValuePair.second;
        // TODO: No need to look up the object twice like this. Use a var.
        if (itr->second == 0) {
            mSumOverPastBuckets.erase(itr);
        }
    }
    mSumPastCounters = 0;
    mOldestBucketIndex = 0;
}

void CountAnomalyTracker::checkAnomaly(int currentCount) {
    // Skip the check if in refractory period.
    if (time(nullptr) < mRefractoryPeriodEndsSec) {
        VLOG("Skipping anomaly check since within refractory period");
void DiscreteAnomalyTracker::addBucketToSum(const shared_ptr<const DimToValMap>& bucket) {
    if (bucket == nullptr) {
        return;
    }
    // For each dimension present in the bucket, add its value to its corresponding sum.
    for (const auto& keyValuePair : *bucket) {
        mSumOverPastBuckets[keyValuePair.first] += keyValuePair.second;
    }
}

    // TODO: Remove these extremely verbose debugging log.
    VLOG("Checking whether %d + %d > %lld",
         mSumPastCounters, currentCount, mAlert.trigger_if_sum_gt());
bool DiscreteAnomalyTracker::detectAnomaly() {
    for (auto itr = mSumOverPastBuckets.begin(); itr != mSumOverPastBuckets.end(); itr++) {
        if (mAlert.has_trigger_if_sum_gt() && itr->second > mAlert.trigger_if_sum_gt()) {
            return true;
        }
    }
    return false;
}

    // Note that this works even if mNumPastBuckets < 1 (since then
    // mSumPastCounters = 0 so the comparison is based only on currentCount).
    if (mAlert.has_trigger_if_sum_gt() &&
            mSumPastCounters + currentCount > mAlert.trigger_if_sum_gt()) {
void DiscreteAnomalyTracker::declareAndDeclareAnomaly() {
    if (detectAnomaly()) {
        declareAnomaly();
    }
}

void CountAnomalyTracker::declareAnomaly() {
void DiscreteAnomalyTracker::declareAnomaly() {
    if (mLastAlarmAtBucketIndex >= 0 && mCurrentBucketIndex - mLastAlarmAtBucketIndex <=
                                        (long long)mAlert.refractory_period_in_buckets()) {
        VLOG("Skipping anomaly check since within refractory period");
        return;
    }
    mAnomalyDeclared++;
    // TODO(guardrail): Consider guarding against too short refractory periods.
    time_t currTime = time(nullptr);
    mRefractoryPeriodEndsSec = currTime + mAlert.refractory_period_secs();

    // TODO: If we had access to the bucket_size_millis, consider calling reset()
    // if (mAlert.refractory_period_secs() > mNumPastBuckets * bucket_size_millis * 1000).
    mLastAlarmAtBucketIndex = mCurrentBucketIndex;

    if (mAlert.has_incidentd_details()) {
        const Alert_IncidentdDetails& incident = mAlert.incidentd_details();
+91 −0
Original line number Diff line number Diff line
@@ -14,10 +14,11 @@
 * limitations under the License.
 */

#ifndef COUNT_ANOMALY_TRACKER_H
#define COUNT_ANOMALY_TRACKER_H
#pragma once

#include <gtest/gtest_prod.h>
#include "frameworks/base/cmds/statsd/src/statsd_config.pb.h" // Alert
#include "stats_util.h" // HashableDimensionKey and DimToValMap

#include <memory> // unique_ptr
#include <stdlib.h>
@@ -26,62 +27,65 @@ namespace android {
namespace os {
namespace statsd {

// TODO: Can probably be used for Count, Value, and Gauge. If so, rename to ValueAnomalyTracker.
// (caveat: currently, the value cannot be negative. Probably fine for P.)
class CountAnomalyTracker {
using std::unordered_map;
using std::shared_ptr;

// This anomaly track assmues that all values are non-negative.
class DiscreteAnomalyTracker {
 public:
    CountAnomalyTracker(const Alert& alert);
    DiscreteAnomalyTracker(const Alert& alert);

    virtual ~DiscreteAnomalyTracker();

    virtual ~CountAnomalyTracker();
    // Adds a new bucket or updates an existing bucket.
    // Bucket index starts from 0.
    void addOrUpdateBucket(std::shared_ptr<const DimToValMap> BucketValues, int64_t bucketIndex);

    // Returns true if detected anomaly for the existing buckets on one or more dimension keys.
    bool detectAnomaly();

    // Adds a new past bucket, holding pastBucketCount, and then advances the
    // present by numberOfBucketsAgo buckets (filling any intervening buckets
    // with 0s).
    // Thus, the newly added bucket (which holds pastBucketCount) is stored
    // numberOfBucketsAgo buckets ago.
    void addPastBucket(int pastBucketCount, time_t numberOfBucketsAgo);
    // Informs incidentd about the detected alert.
    void declareAnomaly();

    // Informs the anomaly tracker of the current bucket's count, so that it can
    // determine whether an anomaly has occurred. This value is not stored.
    void checkAnomaly(int currentCount);
    // Detects the alert and informs the incidentd when applicable.
    void declareAndDeclareAnomaly();

private:
    // statsd_config.proto Alert message that defines this tracker.
    const Alert mAlert;

    // Number of past buckets. One less than the total number of buckets needed
    // for the anomaly detection (since the current bucket is not in the past).
    const size_t mNumPastBuckets;
    // The exisiting bucket list.
    std::vector<shared_ptr<const DimToValMap>> mPastBuckets;

    // Count values for each of the past mNumPastBuckets buckets.
    // TODO: Add dimensions. This parallels the type of CountMetricProducer.mCounter.
    std::unique_ptr<int[]> mPastBuckets;
    // Sum over all existing buckets cached in mPastBuckets.
    DimToValMap mSumOverPastBuckets;

    // Sum over all of mPastBuckets (cached).
    // TODO: Add dimensions. This parallels the type of CountMetricProducer.mCounter.
    //       At that point, mSumPastCounters must never contain entries of 0.
    int mSumPastCounters;
    // Current bucket index of the current anomaly detection window. Bucket index starts from 0.
    int64_t mCurrentBucketIndex = -1;

    // Index of the oldest bucket (i.e. the next bucket to be overwritten).
    size_t mOldestBucketIndex = 0;
    // The bucket index when the last anomaly was declared.
    int64_t mLastAlarmAtBucketIndex = -1;

    // Timestamp that the refractory period (if this anomaly was declared) ends, in seconds.
    // If an anomaly was never declared, set to 0.
    time_t mRefractoryPeriodEndsSec = 0;
    // The total number of declared anomalies.
    int64_t mAnomalyDeclared = 0;

    void declareAnomaly();
    // Add the information in the given bucket to mSumOverPastBuckets.
    void addBucketToSum(const shared_ptr<const DimToValMap>& bucket);

    // Subtract the information in the given bucket from mSumOverPastBuckets
    // and remove any items with value 0.
    void subtractBucketFromSum(const shared_ptr<const DimToValMap>& bucket);

    // Calculates the corresponding index within the circular array.
    size_t index(size_t unsafeIndex) {
        return unsafeIndex % mNumPastBuckets;
    }
    // Calculates the corresponding bucket index within the circular array.
    size_t index(int64_t bucketNum);

    // Resets all data. For use when all the data gets stale.
    void reset();

    FRIEND_TEST(AnomalyTrackerTest, TestConsecutiveBuckets);
    FRIEND_TEST(AnomalyTrackerTest, TestSparseBuckets);
};

}  // namespace statsd
}  // namespace os
}  // namespace android
#endif  // COUNT_ANOMALY_TRACKER_H
+6 −0
Original line number Diff line number Diff line
@@ -165,6 +165,12 @@ static StatsdConfig build_fake_config() {
    KeyMatcher* keyMatcher = metric->add_dimension();
    keyMatcher->set_key(UID_PROCESS_STATE_UID_KEY);

    // Anomaly threshold for background count.
    alert = metric->add_alerts();
    alert->set_number_of_buckets(4);
    alert->set_trigger_if_sum_gt(30);
    alert->set_refractory_period_secs(20);

    // Count process state changes, slice by uid, while SCREEN_IS_OFF
    metric = config.add_count_metric();
    metric->set_metric_id(3);
Loading