Merge "Guardrail for dimension in condition in duration tracker." into pi-dev (014432cb) · Commits · e / os / android_frameworks_base

cmds/statsd/src/guardrail/StatsdStats.cpp

+23 −0

Original line number	Diff line number	Diff line
		@@ -76,6 +76,7 @@ const int FIELD_ID_CONFIG_STATS_MATCHER_STATS = 13;
		const int FIELD_ID_CONFIG_STATS_CONDITION_STATS = 14;
		const int FIELD_ID_CONFIG_STATS_METRIC_STATS = 15;
		const int FIELD_ID_CONFIG_STATS_ALERT_STATS = 16;
		const int FIELD_ID_CONFIG_STATS_METRIC_DIMENSION_IN_CONDITION_STATS = 17;

		const int FIELD_ID_MATCHER_STATS_ID = 1;
		const int FIELD_ID_MATCHER_STATS_COUNT = 2;
		@@ -255,6 +256,20 @@ void StatsdStats::noteMetricDimensionSize(const ConfigKey& key, const int64_t& i
		}
		}

		void StatsdStats::noteMetricDimensionInConditionSize(
		const ConfigKey& key, const int64_t& id, int size) {
		lock_guard<std::mutex> lock(mLock);
		// if name doesn't exist before, it will create the key with count 0.
		auto statsIt = mConfigStats.find(key);
		if (statsIt == mConfigStats.end()) {
		return;
		}
		auto& metricsDimensionMap = statsIt->second->metric_dimension_in_condition_stats;
		if (size > metricsDimensionMap[id]) {
		metricsDimensionMap[id] = size;
		}
		}

		void StatsdStats::noteMatcherMatched(const ConfigKey& key, const int64_t& id) {
		lock_guard<std::mutex> lock(mLock);

		@@ -339,6 +354,7 @@ void StatsdStats::resetInternalLocked() {
		config.second->matcher_stats.clear();
		config.second->condition_stats.clear();
		config.second->metric_stats.clear();
		config.second->metric_dimension_in_condition_stats.clear();
		config.second->alert_stats.clear();
		}
		}
		@@ -504,6 +520,13 @@ void addConfigStatsToProto(const ConfigStats& configStats, ProtoOutputStream* pr
		proto->write(FIELD_TYPE_INT32 \| FIELD_ID_METRIC_STATS_COUNT, pair.second);
		proto->end(tmpToken);
		}
		for (const auto& pair : configStats.metric_dimension_in_condition_stats) {
		uint64_t tmpToken = proto->start(FIELD_TYPE_MESSAGE \| FIELD_COUNT_REPEATED \|
		FIELD_ID_CONFIG_STATS_METRIC_DIMENSION_IN_CONDITION_STATS);
		proto->write(FIELD_TYPE_INT64 \| FIELD_ID_METRIC_STATS_ID, (long long)pair.first);
		proto->write(FIELD_TYPE_INT32 \| FIELD_ID_METRIC_STATS_COUNT, pair.second);
		proto->end(tmpToken);
		}

		for (const auto& pair : configStats.alert_stats) {
		uint64_t tmpToken = proto->start(FIELD_TYPE_MESSAGE \| FIELD_COUNT_REPEATED \|

cmds/statsd/src/guardrail/StatsdStats.h

+19 −0

Original line number	Diff line number	Diff line
		@@ -57,6 +57,12 @@ struct ConfigStats {
		// it means some data has been dropped. The map size is capped by kMaxConfigCount.
		std::map<const int64_t, int> metric_stats;

		// Stores the max number of output tuple of dimensions in condition across dimensions in what
		// when it's bigger than kDimensionKeySizeSoftLimit. When you see the number is
		// kDimensionKeySizeHardLimit +1, it means some data has been dropped. The map size is capped by
		// kMaxConfigCount.
		std::map<const int64_t, int> metric_dimension_in_condition_stats;

		// Stores the number of times an anomaly detection alert has been declared.
		// The map size is capped by kMaxConfigCount.
		std::map<const int64_t, int> alert_stats;
		@@ -183,6 +189,19 @@ public:
		*/
		void noteMetricDimensionSize(const ConfigKey& key, const int64_t& id, int size);


		/**
		* Report the max size of output tuple of dimension in condition across dimensions in what.
		*
		* Note: only report when the metric has an output dimension in condition, and the max tuple
		* count > kDimensionKeySizeSoftLimit.
		*
		* [key]: The config key that this metric belongs to.
		* [id]: The id of the metric.
		* [size]: The output tuple size.
		*/
		void noteMetricDimensionInConditionSize(const ConfigKey& key, const int64_t& id, int size);

		/**
		* Report a matcher has been matched.
		*

cmds/statsd/src/metrics/DurationMetricProducer.cpp

+44 −15

Original line number	Diff line number	Diff line
		@@ -308,11 +308,14 @@ void DurationMetricProducer::onSlicedConditionMayChangeLocked_opt2(bool conditio
		if (mMetric2ConditionLinks.size() == 0 \|\|
		trueDim.contains(linkedConditionDimensionKey)) {
		if (!whatIt.second.empty()) {
		auto newEventKey = MetricDimensionKey(whatIt.first, trueDim);
		if (hitGuardRailLocked(newEventKey)) {
		continue;
		}
		unique_ptr<DurationTracker> newTracker =
		whatIt.second.begin()->second->clone(eventTime);
		if (newTracker != nullptr) {
		newTracker->setEventKey(
		MetricDimensionKey(whatIt.first, trueDim));
		newTracker->setEventKey(newEventKey);
		newTracker->onConditionChanged(true, eventTime);
		whatIt.second[trueDim] = std::move(newTracker);
		}
		@@ -370,11 +373,14 @@ void DurationMetricProducer::onSlicedConditionMayChangeLocked(bool overallCondit
		for (const auto& conditionDimension : conditionDimensionsKeySet) {
		for (auto& whatIt : mCurrentSlicedDurationTrackerMap) {
		if (!whatIt.second.empty()) {
		auto newEventKey = MetricDimensionKey(whatIt.first, conditionDimension);
		if (hitGuardRailLocked(newEventKey)) {
		continue;
		}
		unique_ptr<DurationTracker> newTracker =
		whatIt.second.begin()->second->clone(eventTime);
		if (newTracker != nullptr) {
		newTracker->setEventKey(MetricDimensionKey(
		whatIt.first, conditionDimension));
		newTracker->setEventKey(MetricDimensionKey(newEventKey));
		newTracker->onSlicedConditionMayChange(overallCondition, eventTime);
		whatIt.second[conditionDimension] = std::move(newTracker);
		}
		@@ -397,10 +403,13 @@ void DurationMetricProducer::onSlicedConditionMayChangeLocked(bool overallCondit
		for (const auto& conditionDimension : conditionDimensionsKeys) {
		if (!whatIt.second.empty() &&
		whatIt.second.find(conditionDimension) == whatIt.second.end()) {
		auto newEventKey = MetricDimensionKey(whatIt.first, conditionDimension);
		if (hitGuardRailLocked(newEventKey)) {
		continue;
		}
		auto newTracker = whatIt.second.begin()->second->clone(eventTime);
		if (newTracker != nullptr) {
		newTracker->setEventKey(
		MetricDimensionKey(whatIt.first, conditionDimension));
		newTracker->setEventKey(newEventKey);
		newTracker->onSlicedConditionMayChange(overallCondition, eventTime);
		whatIt.second[conditionDimension] = std::move(newTracker);
		}
		@@ -552,17 +561,37 @@ void DurationMetricProducer::dumpStatesLocked(FILE* out, bool verbose) const {
		}

		bool DurationMetricProducer::hitGuardRailLocked(const MetricDimensionKey& newKey) {
		auto whatIt = mCurrentSlicedDurationTrackerMap.find(newKey.getDimensionKeyInWhat());
		if (whatIt != mCurrentSlicedDurationTrackerMap.end()) {
		auto condIt = whatIt->second.find(newKey.getDimensionKeyInCondition());
		if (condIt != whatIt->second.end()) {
		return false;
		}
		if (whatIt->second.size() > StatsdStats::kDimensionKeySizeSoftLimit - 1) {
		size_t newTupleCount = whatIt->second.size() + 1;
		StatsdStats::getInstance().noteMetricDimensionInConditionSize(
		mConfigKey, mMetricId, newTupleCount);
		// 2. Don't add more tuples, we are above the allowed threshold. Drop the data.
		if (newTupleCount > StatsdStats::kDimensionKeySizeHardLimit) {
		ALOGE("DurationMetric %lld dropping data for condition dimension key %s",
		(long long)mMetricId, newKey.getDimensionKeyInCondition().toString().c_str());
		return true;
		}
		}
		} else {
		// 1. Report the tuple count if the tuple count > soft limit
		if (mCurrentSlicedDurationTrackerMap.size() > StatsdStats::kDimensionKeySizeSoftLimit - 1) {
		size_t newTupleCount = mCurrentSlicedDurationTrackerMap.size() + 1;
		StatsdStats::getInstance().noteMetricDimensionSize(mConfigKey, mMetricId, newTupleCount);
		StatsdStats::getInstance().noteMetricDimensionSize(
		mConfigKey, mMetricId, newTupleCount);
		// 2. Don't add more tuples, we are above the allowed threshold. Drop the data.
		if (newTupleCount > StatsdStats::kDimensionKeySizeHardLimit) {
		ALOGE("DurationMetric %lld dropping data for dimension key %s",
		(long long)mMetricId, newKey.toString().c_str());
		ALOGE("DurationMetric %lld dropping data for what dimension key %s",
		(long long)mMetricId, newKey.getDimensionKeyInWhat().toString().c_str());
		return true;
		}
		}
		}
		return false;
		}

cmds/statsd/src/stats_log.proto

+1 −0

Original line number	Diff line number	Diff line
		@@ -241,6 +241,7 @@ message StatsdStatsReport {
		repeated ConditionStats condition_stats = 14;
		repeated MetricStats metric_stats = 15;
		repeated AlertStats alert_stats = 16;
		repeated MetricStats metric_dimension_in_condition_stats = 17;
		}

		repeated ConfigStats config_stats = 3;