Merge "Update CountAnomalyTracker to handle slicing."

    src/matchers/CombinationLogMatchingTracker.cpp \

    src/matchers/matcher_util.cpp \

    src/matchers/SimpleLogMatchingTracker.cpp \

    src/metrics/CountAnomalyTracker.cpp \

    src/anomaly/DiscreteAnomalyTracker.cpp \

    src/metrics/MetricProducer.cpp \

    src/metrics/EventMetricProducer.cpp \

    src/metrics/CountMetricProducer.cpp \

LOCAL_SRC_FILES := \

    $(statsd_common_src) \

    tests/AnomalyMonitor_test.cpp \

    tests/anomaly/AnomalyTracker_test.cpp \

    tests/ConditionTracker_test.cpp \

    tests/ConfigManager_test.cpp \

    tests/indexed_priority_queue_test.cpp \

    };

};

// TODO: Rename this file to AnomalyAlarmMonitor.

/**

 * Manages alarms for Anomaly Detection.

 */

    unordered_set<sp<const AnomalyAlarm>, SpHash<AnomalyAlarm>> popSoonerThan(

            uint32_t timestampSec);

    // TODO: Function that uses popSoonerThan to get all alarms that have fired, and then

    // iterates over all DurationAnomalyTracker, looking for those alarms. When they're found,

    // have them declareAnomaly on those alarms. This means that DurationAnomalyTracker

    // must be thread-safe (since this is being called on a different thread). There is no

    // worry about missing the alarms (due to them being cancelled after this function being called)

    // because DurationAnomalyTracker guarantees that it checks for anaomlies when it cancels

    // alarms anyway.

    // void declareAnomalies(uint32_t timestampSec);

    /**

     * Returns the projected alarm timestamp that is registered with

     * StatsCompanionService. This may not be equal to the soonest alarm,

#define DEBUG true  // STOPSHIP if true

#include "Log.h"

#include "CountAnomalyTracker.h"

#include "DiscreteAnomalyTracker.h"

#include <time.h>

namespace os {

namespace statsd {

CountAnomalyTracker::CountAnomalyTracker(const Alert& alert)

    : mAlert(alert),

      mNumPastBuckets(alert.number_of_buckets() > 0 ? alert.number_of_buckets() - 1 : 0),

      mPastBuckets(mNumPastBuckets > 0 ? (new int[mNumPastBuckets]) : nullptr) {

    VLOG("CountAnomalyTracker() called");

    if (alert.number_of_buckets() < 1) {

        ALOGE("Cannot create CountAnomalyTracker with %d buckets", alert.number_of_buckets());

DiscreteAnomalyTracker::DiscreteAnomalyTracker(const Alert& alert) : mAlert(alert) {

    VLOG("DiscreteAnomalyTracker() called");

    if (mAlert.number_of_buckets() <= 0) {

        ALOGE("Cannot create DiscreteAnomalyTracker with %lld buckets",

              (long long)mAlert.number_of_buckets());

        return;

    }

    mPastBuckets.resize(mAlert.number_of_buckets());

    reset(); // initialization

}

CountAnomalyTracker::~CountAnomalyTracker() {

    VLOG("~CountAnomalyTracker() called");

DiscreteAnomalyTracker::~DiscreteAnomalyTracker() {

    VLOG("~DiscreteAnomalyTracker() called");

}

void CountAnomalyTracker::addPastBucket(int pastBucketCount,

                                        time_t numberOfBucketsAgo) {

    VLOG("addPastBucket() called.");

    if (numberOfBucketsAgo < 1) {

        ALOGE("Cannot add a past bucket %ld units in past", numberOfBucketsAgo);

        return;

void DiscreteAnomalyTracker::reset() {

    VLOG("reset() called.");

    mPastBuckets.clear();

    mPastBuckets.resize(mAlert.number_of_buckets());

    mSumOverPastBuckets.clear();

    mCurrentBucketIndex = -1;

    mLastAlarmAtBucketIndex = -1;

    mAnomalyDeclared = 0;

}

size_t DiscreteAnomalyTracker::index(int64_t bucketNum) {

    return bucketNum % mAlert.number_of_buckets();

}

    // If past bucket was ancient, just empty out all past info.

    // This always applies if mNumPastBuckets == 0 (i.e. store no past buckets).

    if (numberOfBucketsAgo > (time_t) mNumPastBuckets) {

        reset();

void DiscreteAnomalyTracker::addOrUpdateBucket(std::shared_ptr<const DimToValMap> BucketValues,

                                               int64_t bucketIndex) {

    VLOG("addPastBucket() called.");

    if (bucketIndex <= mCurrentBucketIndex - mAlert.number_of_buckets()) {

        ALOGE("Cannot add a past bucket %lld units in past", (long long)bucketIndex);

        return;

    }

    // Empty out old mPastBuckets[i] values and update mSumPastCounters.

    for (size_t i = mOldestBucketIndex;

                        i < mOldestBucketIndex + numberOfBucketsAgo; i++) {

        mSumPastCounters -= mPastBuckets[index(i)];

        mPastBuckets[index(i)] = 0;

    // Empty out old mPastBuckets[i] values and update mSumOverPastBuckets.

    if (bucketIndex - mCurrentBucketIndex >= mAlert.number_of_buckets()) {

        mPastBuckets.clear();

        mPastBuckets.resize(mAlert.number_of_buckets());

        mSumOverPastBuckets.clear();

    } else {

        for (int64_t i = std::max(

                     0LL, (long long)(mCurrentBucketIndex - mAlert.number_of_buckets() + 1));

             i < bucketIndex - mAlert.number_of_buckets(); i++) {

            const int idx = index(i);

            subtractBucketFromSum(mPastBuckets[idx]);

            mPastBuckets[idx] = nullptr;  // release (but not clear) the old bucket.

        }

    }

    subtractBucketFromSum(mPastBuckets[index(bucketIndex)]);

    mPastBuckets[index(bucketIndex)] = nullptr;  // release (but not clear) the old bucket.

    // Replace the oldest bucket with the new bucket we are adding.

    mPastBuckets[mOldestBucketIndex] = pastBucketCount;

    mSumPastCounters += pastBucketCount;

    mPastBuckets[index(bucketIndex)] = BucketValues;

    addBucketToSum(BucketValues);

    // Advance the oldest bucket index by numberOfBucketsAgo units.

    mOldestBucketIndex = index(mOldestBucketIndex + numberOfBucketsAgo);

    // TODO: Once dimensions are added to mSumPastCounters:

    // iterate through mSumPastCounters and remove any entries that are 0.

    mCurrentBucketIndex = std::max(mCurrentBucketIndex, bucketIndex);

}

void CountAnomalyTracker::reset() {

    VLOG("reset() called.");

    for (size_t i = 0; i < mNumPastBuckets; i++) {

        mPastBuckets[i] = 0;

void DiscreteAnomalyTracker::subtractBucketFromSum(const shared_ptr<const DimToValMap>& bucket) {

    if (bucket == nullptr) {

        return;

    }

    // For each dimension present in the bucket, subtract its value from its corresponding sum.

    for (const auto& keyValuePair : *bucket) {

        auto itr = mSumOverPastBuckets.find(keyValuePair.first);

        if (itr == mSumOverPastBuckets.end()) {

            continue;

        }

        itr->second -= keyValuePair.second;

        // TODO: No need to look up the object twice like this. Use a var.

        if (itr->second == 0) {

            mSumOverPastBuckets.erase(itr);

        }

    }

    mSumPastCounters = 0;

    mOldestBucketIndex = 0;

}

void CountAnomalyTracker::checkAnomaly(int currentCount) {

    // Skip the check if in refractory period.

    if (time(nullptr) < mRefractoryPeriodEndsSec) {

        VLOG("Skipping anomaly check since within refractory period");

void DiscreteAnomalyTracker::addBucketToSum(const shared_ptr<const DimToValMap>& bucket) {

    if (bucket == nullptr) {

        return;

    }

    // For each dimension present in the bucket, add its value to its corresponding sum.

    for (const auto& keyValuePair : *bucket) {

        mSumOverPastBuckets[keyValuePair.first] += keyValuePair.second;

    }

}

    // TODO: Remove these extremely verbose debugging log.

    VLOG("Checking whether %d + %d > %lld",

         mSumPastCounters, currentCount, mAlert.trigger_if_sum_gt());

bool DiscreteAnomalyTracker::detectAnomaly() {

    for (auto itr = mSumOverPastBuckets.begin(); itr != mSumOverPastBuckets.end(); itr++) {

        if (mAlert.has_trigger_if_sum_gt() && itr->second > mAlert.trigger_if_sum_gt()) {

            return true;

        }

    }

    return false;

}

    // Note that this works even if mNumPastBuckets < 1 (since then

    // mSumPastCounters = 0 so the comparison is based only on currentCount).

    if (mAlert.has_trigger_if_sum_gt() &&

            mSumPastCounters + currentCount > mAlert.trigger_if_sum_gt()) {

void DiscreteAnomalyTracker::declareAndDeclareAnomaly() {

    if (detectAnomaly()) {

        declareAnomaly();

    }

}

void CountAnomalyTracker::declareAnomaly() {

void DiscreteAnomalyTracker::declareAnomaly() {

    if (mLastAlarmAtBucketIndex >= 0 && mCurrentBucketIndex - mLastAlarmAtBucketIndex <=

                                        (long long)mAlert.refractory_period_in_buckets()) {

        VLOG("Skipping anomaly check since within refractory period");

        return;

    }

    mAnomalyDeclared++;

    // TODO(guardrail): Consider guarding against too short refractory periods.

    time_t currTime = time(nullptr);

    mRefractoryPeriodEndsSec = currTime + mAlert.refractory_period_secs();

    // TODO: If we had access to the bucket_size_millis, consider calling reset()

    // if (mAlert.refractory_period_secs() > mNumPastBuckets * bucket_size_millis * 1000).

    mLastAlarmAtBucketIndex = mCurrentBucketIndex;

    if (mAlert.has_incidentd_details()) {

        const Alert_IncidentdDetails& incident = mAlert.incidentd_details();

 * limitations under the License.

 */

#ifndef COUNT_ANOMALY_TRACKER_H

#define COUNT_ANOMALY_TRACKER_H

#pragma once

#include <gtest/gtest_prod.h>

#include "frameworks/base/cmds/statsd/src/statsd_config.pb.h" // Alert

#include "stats_util.h" // HashableDimensionKey and DimToValMap

#include <memory> // unique_ptr

#include <stdlib.h>

namespace os {

namespace statsd {

// TODO: Can probably be used for Count, Value, and Gauge. If so, rename to ValueAnomalyTracker.

// (caveat: currently, the value cannot be negative. Probably fine for P.)

class CountAnomalyTracker {

using std::unordered_map;

using std::shared_ptr;

// This anomaly track assmues that all values are non-negative.

class DiscreteAnomalyTracker {

 public:

    CountAnomalyTracker(const Alert& alert);

    DiscreteAnomalyTracker(const Alert& alert);

    virtual ~DiscreteAnomalyTracker();

    virtual ~CountAnomalyTracker();

    // Adds a new bucket or updates an existing bucket.

    // Bucket index starts from 0.

    void addOrUpdateBucket(std::shared_ptr<const DimToValMap> BucketValues, int64_t bucketIndex);

    // Returns true if detected anomaly for the existing buckets on one or more dimension keys.

    bool detectAnomaly();

    // Adds a new past bucket, holding pastBucketCount, and then advances the

    // present by numberOfBucketsAgo buckets (filling any intervening buckets

    // with 0s).

    // Thus, the newly added bucket (which holds pastBucketCount) is stored

    // numberOfBucketsAgo buckets ago.

    void addPastBucket(int pastBucketCount, time_t numberOfBucketsAgo);

    // Informs incidentd about the detected alert.

    void declareAnomaly();

    // Informs the anomaly tracker of the current bucket's count, so that it can

    // determine whether an anomaly has occurred. This value is not stored.

    void checkAnomaly(int currentCount);

    // Detects the alert and informs the incidentd when applicable.

    void declareAndDeclareAnomaly();

private:

    // statsd_config.proto Alert message that defines this tracker.

    const Alert mAlert;

    // Number of past buckets. One less than the total number of buckets needed

    // for the anomaly detection (since the current bucket is not in the past).

    const size_t mNumPastBuckets;

    // The exisiting bucket list.

    std::vector<shared_ptr<const DimToValMap>> mPastBuckets;

    // Count values for each of the past mNumPastBuckets buckets.

    // TODO: Add dimensions. This parallels the type of CountMetricProducer.mCounter.

    std::unique_ptr<int[]> mPastBuckets;

    // Sum over all existing buckets cached in mPastBuckets.

    DimToValMap mSumOverPastBuckets;

    // Sum over all of mPastBuckets (cached).

    // TODO: Add dimensions. This parallels the type of CountMetricProducer.mCounter.

    //       At that point, mSumPastCounters must never contain entries of 0.

    int mSumPastCounters;

    // Current bucket index of the current anomaly detection window. Bucket index starts from 0.

    int64_t mCurrentBucketIndex = -1;

    // Index of the oldest bucket (i.e. the next bucket to be overwritten).

    size_t mOldestBucketIndex = 0;

    // The bucket index when the last anomaly was declared.

    int64_t mLastAlarmAtBucketIndex = -1;

    // Timestamp that the refractory period (if this anomaly was declared) ends, in seconds.

    // If an anomaly was never declared, set to 0.

    time_t mRefractoryPeriodEndsSec = 0;

    // The total number of declared anomalies.

    int64_t mAnomalyDeclared = 0;

    void declareAnomaly();

    // Add the information in the given bucket to mSumOverPastBuckets.

    void addBucketToSum(const shared_ptr<const DimToValMap>& bucket);

    // Subtract the information in the given bucket from mSumOverPastBuckets

    // and remove any items with value 0.

    void subtractBucketFromSum(const shared_ptr<const DimToValMap>& bucket);

    // Calculates the corresponding index within the circular array.

    size_t index(size_t unsafeIndex) {

        return unsafeIndex % mNumPastBuckets;

    }

    // Calculates the corresponding bucket index within the circular array.

    size_t index(int64_t bucketNum);

    // Resets all data. For use when all the data gets stale.

    void reset();

    FRIEND_TEST(AnomalyTrackerTest, TestConsecutiveBuckets);

    FRIEND_TEST(AnomalyTrackerTest, TestSparseBuckets);

};

}  // namespace statsd

}  // namespace os

}  // namespace android

#endif  // COUNT_ANOMALY_TRACKER_H

    KeyMatcher* keyMatcher = metric->add_dimension();

    keyMatcher->set_key(UID_PROCESS_STATE_UID_KEY);

    // Anomaly threshold for background count.

    alert = metric->add_alerts();

    alert->set_number_of_buckets(4);

    alert->set_trigger_if_sum_gt(30);

    alert->set_refractory_period_secs(20);

    // Count process state changes, slice by uid, while SCREEN_IS_OFF

    metric = config.add_count_metric();

    metric->set_metric_id(3);