Splits AnomalyTracker into two files (857aaa52) · Commits · e / os / android_frameworks_base

cmds/statsd/Android.mk

+1 −0

Original line number	Diff line number	Diff line
		@@ -22,6 +22,7 @@ statsd_common_src := \
		src/atoms.proto \
		src/anomaly/AnomalyMonitor.cpp \
		src/anomaly/AnomalyTracker.cpp \
		src/anomaly/DurationAnomalyTracker.cpp \
		src/condition/CombinationConditionTracker.cpp \
		src/condition/condition_util.cpp \
		src/condition/SimpleConditionTracker.cpp \

cmds/statsd/src/StatsLogProcessor.cpp

+3 −0

Original line number	Diff line number	Diff line
		@@ -81,6 +81,9 @@ StatsLogProcessor::~StatsLogProcessor() {
		void StatsLogProcessor::onAnomalyAlarmFired(
		const uint64_t timestampNs,
		unordered_set<sp<const AnomalyAlarm>, SpHash<AnomalyAlarm>> anomalySet) {
		// TODO: This is a thread-safety issue. mMetricsManagers could change under our feet.
		// TODO: Solution? Lock everything! :(
		// TODO: Question: Can we replace the other lock (broadcast), or do we need to supplement it?
		for (const auto& itr : mMetricsManagers) {
		itr.second->onAnomalyAlarmFired(timestampNs, anomalySet);
		}

cmds/statsd/src/anomaly/AnomalyTracker.cpp

+7 −86

Original line number	Diff line number	Diff line
		@@ -30,8 +30,6 @@ namespace android {
		namespace os {
		namespace statsd {

		// TODO: Separate DurationAnomalyTracker as a separate subclass and let each MetricProducer
		// decide and let which one it wants.
		// TODO: Get rid of bucketNumbers, and return to the original circular array method.
		AnomalyTracker::AnomalyTracker(const Alert& alert, const ConfigKey& configKey)
		: mAlert(alert),
		@@ -52,7 +50,6 @@ AnomalyTracker::AnomalyTracker(const Alert& alert, const ConfigKey& configKey)

		AnomalyTracker::~AnomalyTracker() {
		VLOG("~AnomalyTracker() called");
		stopAllAlarms();
		}

		void AnomalyTracker::resetStorage() {
		@@ -61,8 +58,6 @@ void AnomalyTracker::resetStorage() {
		// Excludes the current bucket.
		mPastBuckets.resize(mNumOfPastBuckets);
		mSumOverPastBuckets.clear();

		if (!mAlarms.empty()) VLOG("AnomalyTracker.resetStorage() called but mAlarms is NOT empty!");
		}

		size_t AnomalyTracker::index(int64_t bucketNum) const {
		@@ -205,23 +200,22 @@ void AnomalyTracker::declareAnomaly(const uint64_t& timestampNs) {
		return;
		}
		// TODO(guardrail): Consider guarding against too short refractory periods.
		mLastAlarmTimestampNs = timestampNs;

		mLastAnomalyTimestampNs = timestampNs;

		// TODO: If we had access to the bucket_size_millis, consider calling resetStorage()
		// if (mAlert.refractory_period_secs() > mNumOfPastBuckets * bucketSizeNs) { resetStorage(); }

		if (mAlert.has_incidentd_details()) {
		if (mAlert.has_name()) {
		ALOGW("An anomaly (%s) has occurred! Informing incidentd.",
		ALOGI("An anomaly (%s) has occurred! Informing incidentd.",
		mAlert.name().c_str());
		} else {
		// TODO: Can construct a name based on the criteria (and/or relay the criteria).
		ALOGW("An anomaly (nameless) has occurred! Informing incidentd.");
		ALOGI("An anomaly (nameless) has occurred! Informing incidentd.");
		}
		informIncidentd();
		} else {
		ALOGW("An anomaly has occurred! (But informing incidentd not requested.)");
		ALOGI("An anomaly has occurred! (But informing incidentd not requested.)");
		}

		StatsdStats::getInstance().noteAnomalyDeclared(mConfigKey, mAlert.name());
		@@ -230,20 +224,6 @@ void AnomalyTracker::declareAnomaly(const uint64_t& timestampNs) {
		mConfigKey.GetName().c_str(), mAlert.name().c_str());
		}

		void AnomalyTracker::declareAnomalyIfAlarmExpired(const HashableDimensionKey& dimensionKey,
		const uint64_t& timestampNs) {
		auto itr = mAlarms.find(dimensionKey);
		if (itr == mAlarms.end()) {
		return;
		}

		if (itr->second != nullptr &&
		static_cast<uint32_t>(timestampNs / NS_PER_SEC) >= itr->second->timestampSec) {
		declareAnomaly(timestampNs);
		stopAlarm(dimensionKey);
		}
		}

		void AnomalyTracker::detectAndDeclareAnomaly(const uint64_t& timestampNs,
		const int64_t& currBucketNum,
		const HashableDimensionKey& key,
		@@ -261,68 +241,9 @@ void AnomalyTracker::detectAndDeclareAnomaly(const uint64_t& timestampNs,
		}
		}

		void AnomalyTracker::startAlarm(const HashableDimensionKey& dimensionKey,
		const uint64_t& timestampNs) {
		uint32_t timestampSec = static_cast<uint32_t>(timestampNs / NS_PER_SEC);
		if (isInRefractoryPeriod(timestampNs)) {
		VLOG("Skipping setting anomaly alarm since it'd fall in the refractory period");
		return;
		}

		sp<const AnomalyAlarm> alarm = new AnomalyAlarm{timestampSec};
		mAlarms.insert({dimensionKey, alarm});
		if (mAnomalyMonitor != nullptr) {
		mAnomalyMonitor->add(alarm);
		}
		}

		void AnomalyTracker::stopAlarm(const HashableDimensionKey& dimensionKey) {
		auto itr = mAlarms.find(dimensionKey);
		if (itr != mAlarms.end()) {
		mAlarms.erase(dimensionKey);
		if (mAnomalyMonitor != nullptr) {
		mAnomalyMonitor->remove(itr->second);
		}
		}
		}

		void AnomalyTracker::stopAllAlarms() {
		std::set<HashableDimensionKey> keys;
		for (auto itr = mAlarms.begin(); itr != mAlarms.end(); ++itr) {
		keys.insert(itr->first);
		}
		for (auto key : keys) {
		stopAlarm(key);
		}
		}

		bool AnomalyTracker::isInRefractoryPeriod(const uint64_t& timestampNs) {
		return mLastAlarmTimestampNs >= 0 &&
		timestampNs - mLastAlarmTimestampNs <= mAlert.refractory_period_secs() * NS_PER_SEC;
		}

		void AnomalyTracker::informAlarmsFired(const uint64_t& timestampNs,
		unordered_set<sp<const AnomalyAlarm>, SpHash<AnomalyAlarm>>& firedAlarms) {

		if (firedAlarms.empty() \|\| mAlarms.empty()) return;
		// Find the intersection of firedAlarms and mAlarms.
		// The for loop is inefficient, since it loops over all keys, but that's okay since it is very
		// seldomly called. The alternative would be having AnomalyAlarms store information about the
		// AnomalyTracker and key, but that's a lot of data overhead to speed up something that is
		// rarely ever called.
		unordered_map<HashableDimensionKey, sp<const AnomalyAlarm>> matchedAlarms;
		for (const auto& kv : mAlarms) {
		if (firedAlarms.count(kv.second) > 0) {
		matchedAlarms.insert({kv.first, kv.second});
		}
		}

		// Now declare each of these alarms to have fired.
		for (const auto& kv : matchedAlarms) {
		declareAnomaly(timestampNs /* TODO: , kv.first */);
		mAlarms.erase(kv.first);
		firedAlarms.erase(kv.second); // No one else can also own it, so we're done with it.
		}
		bool AnomalyTracker::isInRefractoryPeriod(const uint64_t& timestampNs) const {
		return mLastAnomalyTimestampNs >= 0 &&
		timestampNs - mLastAnomalyTimestampNs <= mAlert.refractory_period_secs() * NS_PER_SEC;
		}

		void AnomalyTracker::informIncidentd() {

cmds/statsd/src/anomaly/AnomalyTracker.h

+15 −40

Original line number	Diff line number	Diff line
		@@ -61,23 +61,11 @@ public:
		const HashableDimensionKey& key,
		const int64_t& currentBucketValue);

		// Starts the alarm at the given timestamp.
		void startAlarm(const HashableDimensionKey& dimensionKey, const uint64_t& eventTime);
		// Stops the alarm.
		void stopAlarm(const HashableDimensionKey& dimensionKey);

		// Stop all the alarms owned by this tracker.
		void stopAllAlarms();

		// Init the anmaly monitor which is shared across anomaly trackers.
		inline void setAnomalyMonitor(const sp<AnomalyMonitor>& anomalyMonitor) {
		mAnomalyMonitor = anomalyMonitor;
		// Init the AnomalyMonitor which is shared across anomaly trackers.
		virtual void setAnomalyMonitor(const sp<AnomalyMonitor>& anomalyMonitor) {
		return; // Base AnomalyTracker class has no need for the AnomalyMonitor.
		}

		// Declares the anomaly when the alarm expired given the current timestamp.
		void declareAnomalyIfAlarmExpired(const HashableDimensionKey& dimensionKey,
		const uint64_t& timestampNs);

		// Helper function to return the sum value of past buckets at given dimension.
		int64_t getSumOverPastBuckets(const HashableDimensionKey& key) const;

		@@ -89,9 +77,9 @@ public:
		return mAlert.trigger_if_sum_gt();
		}

		// Helper function to return the last alarm timestamp.
		inline int64_t getLastAlarmTimestampNs() const {
		return mLastAlarmTimestampNs;
		// Helper function to return the timestamp of the last detected anomaly.
		inline int64_t getLastAnomalyTimestampNs() const {
		return mLastAnomalyTimestampNs;
		}

		inline int getNumOfPastBuckets() const {
		@@ -100,15 +88,12 @@ public:

		// Declares an anomaly for each alarm in firedAlarms that belongs to this AnomalyTracker,
		// and removes it from firedAlarms. Does NOT remove the alarm from the AnomalyMonitor.
		// TODO: This will actually be called from a different thread, so make it thread-safe!
		// TODO: Consider having AnomalyMonitor have a reference to each relevant MetricProducer
		// instead of calling it from a chain starting at StatsLogProcessor.
		void informAlarmsFired(const uint64_t& timestampNs,
		unordered_set<sp<const AnomalyAlarm>, SpHash<AnomalyAlarm>>& firedAlarms);
		virtual void informAlarmsFired(const uint64_t& timestampNs,
		unordered_set<sp<const AnomalyAlarm>, SpHash<AnomalyAlarm>>& firedAlarms) {
		return; // The base AnomalyTracker class doesn't have alarms.
		}

		protected:
		void flushPastBuckets(const int64_t& currBucketNum);

		// statsd_config.proto Alert message that defines this tracker.
		const Alert mAlert;

		@@ -119,13 +104,6 @@ protected:
		// for the anomaly detection (since the current bucket is not in the past).
		int mNumOfPastBuckets;

		// The alarms owned by this tracker. The alarm monitor also shares the alarm pointers when they
		// are still active.
		std::unordered_map<HashableDimensionKey, sp<const AnomalyAlarm>> mAlarms;

		// Anomaly alarm monitor.
		sp<AnomalyMonitor> mAnomalyMonitor;

		// The exisiting bucket list.
		std::vector<shared_ptr<DimToValMap>> mPastBuckets;

		@@ -136,7 +114,9 @@ protected:
		int64_t mMostRecentBucketNum = -1;

		// The timestamp when the last anomaly was declared.
		int64_t mLastAlarmTimestampNs = -1;
		int64_t mLastAnomalyTimestampNs = -1;

		void flushPastBuckets(const int64_t& currBucketNum);

		// Add the information in the given bucket to mSumOverPastBuckets.
		void addBucketToSum(const shared_ptr<DimToValMap>& bucket);
		@@ -145,13 +125,13 @@ protected:
		// and remove any items with value 0.
		void subtractBucketFromSum(const shared_ptr<DimToValMap>& bucket);

		bool isInRefractoryPeriod(const uint64_t& timestampNs);
		bool isInRefractoryPeriod(const uint64_t& timestampNs) const;

		// Calculates the corresponding bucket index within the circular array.
		size_t index(int64_t bucketNum) const;

		// Resets all bucket data. For use when all the data gets stale.
		void resetStorage();
		virtual void resetStorage();

		// Informs the incident service that an anomaly has occurred.
		void informIncidentd();
		@@ -160,11 +140,6 @@ protected:
		FRIEND_TEST(AnomalyTrackerTest, TestSparseBuckets);
		FRIEND_TEST(GaugeMetricProducerTest, TestAnomalyDetection);
		FRIEND_TEST(CountMetricProducerTest, TestAnomalyDetection);
		FRIEND_TEST(OringDurationTrackerTest, TestPredictAnomalyTimestamp);
		FRIEND_TEST(OringDurationTrackerTest, TestAnomalyDetection);
		FRIEND_TEST(MaxDurationTrackerTest, TestAnomalyDetection);
		FRIEND_TEST(MaxDurationTrackerTest, TestAnomalyDetection);
		FRIEND_TEST(OringDurationTrackerTest, TestAnomalyDetection);
		};

		} // namespace statsd

cmds/statsd/src/anomaly/DurationAnomalyTracker.cpp

0 → 100644

+115 −0

Original line number	Diff line number	Diff line
		/*
		* Copyright (C) 2017 The Android Open Source Project
		*
		* Licensed under the Apache License, Version 2.0 (the "License");
		* you may not use this file except in compliance with the License.
		* You may obtain a copy of the License at
		*
		* http://www.apache.org/licenses/LICENSE-2.0
		*
		* Unless required by applicable law or agreed to in writing, software
		* distributed under the License is distributed on an "AS IS" BASIS,
		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
		* See the License for the specific language governing permissions and
		* limitations under the License.
		*/

		#define DEBUG true // STOPSHIP if true
		#include "Log.h"

		#include "DurationAnomalyTracker.h"
		#include "guardrail/StatsdStats.h"

		namespace android {
		namespace os {
		namespace statsd {

		DurationAnomalyTracker::DurationAnomalyTracker(const Alert& alert, const ConfigKey& configKey)
		: AnomalyTracker(alert, configKey) {
		}

		DurationAnomalyTracker::~DurationAnomalyTracker() {
		stopAllAlarms();
		}

		void DurationAnomalyTracker::resetStorage() {
		AnomalyTracker::resetStorage();
		if (!mAlarms.empty()) VLOG("AnomalyTracker.resetStorage() called but mAlarms is NOT empty!");
		}

		void DurationAnomalyTracker::declareAnomalyIfAlarmExpired(const HashableDimensionKey& dimensionKey,
		const uint64_t& timestampNs) {
		auto itr = mAlarms.find(dimensionKey);
		if (itr == mAlarms.end()) {
		return;
		}

		if (itr->second != nullptr &&
		static_cast<uint32_t>(timestampNs / NS_PER_SEC) >= itr->second->timestampSec) {
		declareAnomaly(timestampNs);
		stopAlarm(dimensionKey);
		}
		}

		void DurationAnomalyTracker::startAlarm(const HashableDimensionKey& dimensionKey,
		const uint64_t& timestampNs) {

		uint32_t timestampSec = static_cast<uint32_t>(timestampNs / NS_PER_SEC);
		if (isInRefractoryPeriod(timestampNs)) {
		VLOG("Skipping setting anomaly alarm since it'd fall in the refractory period");
		return;
		}
		sp<const AnomalyAlarm> alarm = new AnomalyAlarm{timestampSec};
		mAlarms.insert({dimensionKey, alarm});
		if (mAnomalyMonitor != nullptr) {
		mAnomalyMonitor->add(alarm);
		}
		}

		void DurationAnomalyTracker::stopAlarm(const HashableDimensionKey& dimensionKey) {
		auto itr = mAlarms.find(dimensionKey);
		if (itr != mAlarms.end()) {
		mAlarms.erase(dimensionKey);
		if (mAnomalyMonitor != nullptr) {
		mAnomalyMonitor->remove(itr->second);
		}
		}
		}

		void DurationAnomalyTracker::stopAllAlarms() {
		std::set<HashableDimensionKey> keys;
		for (auto itr = mAlarms.begin(); itr != mAlarms.end(); ++itr) {
		keys.insert(itr->first);
		}
		for (auto key : keys) {
		stopAlarm(key);
		}
		}

		void DurationAnomalyTracker::informAlarmsFired(const uint64_t& timestampNs,
		unordered_set<sp<const AnomalyAlarm>, SpHash<AnomalyAlarm>>& firedAlarms) {

		if (firedAlarms.empty() \|\| mAlarms.empty()) return;
		// Find the intersection of firedAlarms and mAlarms.
		// The for loop is inefficient, since it loops over all keys, but that's okay since it is very
		// seldomly called. The alternative would be having AnomalyAlarms store information about the
		// DurationAnomalyTracker and key, but that's a lot of data overhead to speed up something that is
		// rarely ever called.
		unordered_map<HashableDimensionKey, sp<const AnomalyAlarm>> matchedAlarms;
		for (const auto& kv : mAlarms) {
		if (firedAlarms.count(kv.second) > 0) {
		matchedAlarms.insert({kv.first, kv.second});
		}
		}

		// Now declare each of these alarms to have fired.
		for (const auto& kv : matchedAlarms) {
		declareAnomaly(timestampNs /* TODO: , kv.first */);
		mAlarms.erase(kv.first);
		firedAlarms.erase(kv.second); // No one else can also own it, so we're done with it.
		}
		}

		} // namespace statsd
		} // namespace os
		} // namespace android