Merge "Duration tracker optimization." (9c53ea3a) · Commits · e / os / android_frameworks_base

cmds/statsd/src/FieldValue.h

+12 −0

Original line number	Diff line number	Diff line
		@@ -217,6 +217,14 @@ struct Matcher {
		const Field mMatcher;
		const int32_t mMask;

		inline const Field& getMatcher() const {
		return mMatcher;
		}

		inline int32_t getMask() const {
		return mMask;
		}

		bool hasAnyPositionMatcher(int* prefix) const {
		if (mMatcher.getDepth() == 2 && mMatcher.getRawPosAtDepth(2) == 0) {
		(*prefix) = mMatcher.getPrefix(2);
		@@ -224,6 +232,10 @@ struct Matcher {
		}
		return false;
		}

		inline bool operator!=(const Matcher& that) const {
		return mMatcher != that.getMatcher() \|\| mMask != that.getMask();
		};
		};

		/**

cmds/statsd/src/HashableDimensionKey.cpp

+3 −2

Original line number	Diff line number	Diff line
		@@ -166,10 +166,11 @@ void filterGaugeValues(const std::vector<Matcher>& matcherFields,
		}
		}

		void getDimensionForCondition(const LogEvent& event, Metric2Condition links,
		void getDimensionForCondition(const std::vector<FieldValue>& eventValues,
		const Metric2Condition& links,
		vector<HashableDimensionKey>* conditionDimension) {
		// Get the dimension first by using dimension from what.
		filterValues(links.metricFields, event.getValues(), conditionDimension);
		filterValues(links.metricFields, eventValues, conditionDimension);

		// Then replace the field with the dimension from condition.
		for (auto& dim : *conditionDimension) {

cmds/statsd/src/HashableDimensionKey.h

+2 −1

Original line number	Diff line number	Diff line
		@@ -144,7 +144,8 @@ bool filterValues(const std::vector<Matcher>& matcherFields, const std::vector<F
		void filterGaugeValues(const std::vector<Matcher>& matchers, const std::vector<FieldValue>& values,
		std::vector<FieldValue>* output);

		void getDimensionForCondition(const LogEvent& event, Metric2Condition links,
		void getDimensionForCondition(const std::vector<FieldValue>& eventValues,
		const Metric2Condition& links,
		std::vector<HashableDimensionKey>* conditionDimension);

		} // namespace statsd

cmds/statsd/src/metrics/DurationMetricProducer.cpp

+226 −60

Original line number	Diff line number	Diff line
		@@ -101,6 +101,18 @@ DurationMetricProducer::DurationMetricProducer(const ConfigKey& key, const Durat
		}
		mConditionSliced = (metric.links().size() > 0) \|\| (mDimensionsInCondition.size() > 0);

		if (mDimensionsInWhat.size() == mInternalDimensions.size()) {
		bool mUseWhatDimensionAsInternalDimension = true;
		for (size_t i = 0; mUseWhatDimensionAsInternalDimension &&
		i < mDimensionsInWhat.size(); ++i) {
		if (mDimensionsInWhat[i] != mInternalDimensions[i]) {
		mUseWhatDimensionAsInternalDimension = false;
		}
		}
		} else {
		mUseWhatDimensionAsInternalDimension = false;
		}

		VLOG("metric %lld created. bucket size %lld start_time: %lld", (long long)metric.id(),
		(long long)mBucketSizeNs, (long long)mStartTimeNs);
		}
		@@ -141,29 +153,56 @@ void DurationMetricProducer::onSlicedConditionMayChangeLocked(const uint64_t eve
		flushIfNeededLocked(eventTime);

		// Now for each of the on-going event, check if the condition has changed for them.
		for (auto& pair : mCurrentSlicedDurationTrackerMap) {
		for (auto& whatIt : mCurrentSlicedDurationTrackerMap) {
		for (auto& pair : whatIt.second) {
		pair.second->onSlicedConditionMayChange(eventTime);
		}
		}

		if (mDimensionsInCondition.empty()) {
		return;
		}

		if (mMetric2ConditionLinks.empty()) {
		std::unordered_set<HashableDimensionKey> conditionDimensionsKeySet;
		mWizard->getMetConditionDimension(mConditionTrackerIndex, mDimensionsInCondition,
		&conditionDimensionsKeySet);
		for (const auto& whatIt : mCurrentSlicedDurationTrackerMap) {
		for (const auto& pair : whatIt.second) {
		conditionDimensionsKeySet.erase(pair.first);
		}
		}
		for (const auto& conditionDimension : conditionDimensionsKeySet) {
		for (auto& whatIt : mCurrentSlicedDurationTrackerMap) {
		if (!whatIt.second.empty()) {
		unique_ptr<DurationTracker> newTracker =
		whatIt.second.begin()->second->clone(eventTime);
		newTracker->setEventKey(MetricDimensionKey(whatIt.first, conditionDimension));
		newTracker->onSlicedConditionMayChange(eventTime);
		whatIt.second[conditionDimension] = std::move(newTracker);
		}
		}
		}
		} else {
		for (auto& whatIt : mCurrentSlicedDurationTrackerMap) {
		ConditionKey conditionKey;
		for (const auto& link : mMetric2ConditionLinks) {
		getDimensionForCondition(whatIt.first.getValues(), link,
		&conditionKey[link.conditionId]);
		}
		std::unordered_set<HashableDimensionKey> conditionDimensionsKeys;
		mWizard->query(mConditionTrackerIndex, conditionKey, mDimensionsInCondition,
		&conditionDimensionsKeys);

		for (auto& pair : mCurrentSlicedDurationTrackerMap) {
		conditionDimensionsKeySet.erase(pair.first.getDimensionKeyInCondition());
		for (const auto& conditionDimension : conditionDimensionsKeys) {
		if (!whatIt.second.empty() &&
		whatIt.second.find(conditionDimension) == whatIt.second.end()) {
		auto newTracker = whatIt.second.begin()->second->clone(eventTime);
		newTracker->setEventKey(MetricDimensionKey(whatIt.first, conditionDimension));
		newTracker->onSlicedConditionMayChange(eventTime);
		whatIt.second[conditionDimension] = std::move(newTracker);
		}
		std::unordered_set<MetricDimensionKey> newKeys;
		for (const auto& conditionDimensionsKey : conditionDimensionsKeySet) {
		for (auto& pair : mCurrentSlicedDurationTrackerMap) {
		auto newKey =
		MetricDimensionKey(pair.first.getDimensionKeyInWhat(), conditionDimensionsKey);
		if (newKeys.find(newKey) == newKeys.end()) {
		mCurrentSlicedDurationTrackerMap[newKey] = pair.second->clone(eventTime);
		mCurrentSlicedDurationTrackerMap[newKey]->setEventKey(newKey);
		mCurrentSlicedDurationTrackerMap[newKey]->onSlicedConditionMayChange(eventTime);
		}
		newKeys.insert(newKey);
		}
		}
		}
		@@ -175,10 +214,12 @@ void DurationMetricProducer::onConditionChangedLocked(const bool conditionMet,
		flushIfNeededLocked(eventTime);
		// TODO: need to populate the condition change time from the event which triggers the condition
		// change, instead of using current time.
		for (auto& pair : mCurrentSlicedDurationTrackerMap) {
		for (auto& whatIt : mCurrentSlicedDurationTrackerMap) {
		for (auto& pair : whatIt.second) {
		pair.second->onConditionChanged(conditionMet, eventTime);
		}
		}
		}

		void DurationMetricProducer::onDumpReportLocked(const uint64_t dumpTimeNs,
		ProtoOutputStream* protoOutput) {
		@@ -241,15 +282,22 @@ void DurationMetricProducer::flushIfNeededLocked(const uint64_t& eventTimeNs) {
		return;
		}
		VLOG("flushing...........");
		for (auto it = mCurrentSlicedDurationTrackerMap.begin();
		it != mCurrentSlicedDurationTrackerMap.end();) {
		for (auto whatIt = mCurrentSlicedDurationTrackerMap.begin();
		whatIt != mCurrentSlicedDurationTrackerMap.end();) {
		for (auto it = whatIt->second.begin(); it != whatIt->second.end();) {
		if (it->second->flushIfNeeded(eventTimeNs, &mPastBuckets)) {
		VLOG("erase bucket for key %s", it->first.c_str());
		it = mCurrentSlicedDurationTrackerMap.erase(it);
		VLOG("erase bucket for key %s %s", whatIt->first.c_str(), it->first.c_str());
		it = whatIt->second.erase(it);
		} else {
		++it;
		}
		}
		if (whatIt->second.empty()) {
		whatIt = mCurrentSlicedDurationTrackerMap.erase(whatIt);
		} else {
		whatIt++;
		}
		}

		int numBucketsForward = 1 + (eventTimeNs - currentBucketEndTimeNs) / mBucketSizeNs;
		mCurrentBucketStartTimeNs = currentBucketEndTimeNs + (numBucketsForward - 1) * mBucketSizeNs;
		@@ -257,15 +305,22 @@ void DurationMetricProducer::flushIfNeededLocked(const uint64_t& eventTimeNs) {
		}

		void DurationMetricProducer::flushCurrentBucketLocked(const uint64_t& eventTimeNs) {
		for (auto it = mCurrentSlicedDurationTrackerMap.begin();
		it != mCurrentSlicedDurationTrackerMap.end();) {
		for (auto whatIt = mCurrentSlicedDurationTrackerMap.begin();
		whatIt != mCurrentSlicedDurationTrackerMap.end();) {
		for (auto it = whatIt->second.begin(); it != whatIt->second.end();) {
		if (it->second->flushCurrentBucket(eventTimeNs, &mPastBuckets)) {
		VLOG("erase bucket for key %s", it->first.c_str());
		it = mCurrentSlicedDurationTrackerMap.erase(it);
		VLOG("erase bucket for key %s %s", whatIt->first.c_str(), it->first.c_str());
		it = whatIt->second.erase(it);
		} else {
		++it;
		}
		}
		if (whatIt->second.empty()) {
		whatIt = mCurrentSlicedDurationTrackerMap.erase(whatIt);
		} else {
		whatIt++;
		}
		}
		}

		void DurationMetricProducer::dumpStatesLocked(FILE* out, bool verbose) const {
		@@ -276,18 +331,16 @@ void DurationMetricProducer::dumpStatesLocked(FILE* out, bool verbose) const {
		fprintf(out, "DurationMetric %lld dimension size %lu\n", (long long)mMetricId,
		(unsigned long)mCurrentSlicedDurationTrackerMap.size());
		if (verbose) {
		for (const auto& slice : mCurrentSlicedDurationTrackerMap) {
		fprintf(out, "\t%s\n", slice.first.c_str());
		for (const auto& whatIt : mCurrentSlicedDurationTrackerMap) {
		for (const auto& slice : whatIt.second) {
		fprintf(out, "\t%s\t%s\n", whatIt.first.c_str(), slice.first.c_str());
		slice.second->dumpStates(out, verbose);
		}
		}
		}
		}

		bool DurationMetricProducer::hitGuardRailLocked(const MetricDimensionKey& newKey) {
		// the key is not new, we are good.
		if (mCurrentSlicedDurationTrackerMap.find(newKey) != mCurrentSlicedDurationTrackerMap.end()) {
		return false;
		}
		// 1. Report the tuple count if the tuple count > soft limit
		if (mCurrentSlicedDurationTrackerMap.size() > StatsdStats::kDimensionKeySizeSoftLimit - 1) {
		size_t newTupleCount = mCurrentSlicedDurationTrackerMap.size() + 1;
		@@ -302,48 +355,161 @@ bool DurationMetricProducer::hitGuardRailLocked(const MetricDimensionKey& newKey
		return false;
		}

		void DurationMetricProducer::handleStartEvent(const MetricDimensionKey& eventKey,
		const ConditionKey& conditionKeys,
		bool condition, const LogEvent& event) {
		const auto& whatKey = eventKey.getDimensionKeyInWhat();
		const auto& condKey = eventKey.getDimensionKeyInCondition();

		auto whatIt = mCurrentSlicedDurationTrackerMap.find(whatKey);
		if (whatIt == mCurrentSlicedDurationTrackerMap.end()) {
		if (hitGuardRailLocked(eventKey)) {
		return;
		}
		mCurrentSlicedDurationTrackerMap[whatKey][condKey] = createDurationTracker(eventKey);
		} else {
		if (whatIt->second.find(condKey) == whatIt->second.end()) {
		if (hitGuardRailLocked(eventKey)) {
		return;
		}
		mCurrentSlicedDurationTrackerMap[whatKey][condKey] = createDurationTracker(eventKey);
		}
		}

		auto it = mCurrentSlicedDurationTrackerMap.find(whatKey)->second.find(condKey);
		if (mUseWhatDimensionAsInternalDimension) {
		it->second->noteStart(whatKey, condition,
		event.GetElapsedTimestampNs(), conditionKeys);
		return;
		}

		std::vector<HashableDimensionKey> values;
		filterValues(mInternalDimensions, event.getValues(), &values);
		if (values.empty()) {
		it->second->noteStart(DEFAULT_DIMENSION_KEY, condition,
		event.GetElapsedTimestampNs(), conditionKeys);
		} else {
		for (const auto& value : values) {
		it->second->noteStart(value, condition, event.GetElapsedTimestampNs(), conditionKeys);
		}
		}

		}

		void DurationMetricProducer::onMatchedLogEventInternalLocked(
		const size_t matcherIndex, const MetricDimensionKey& eventKey,
		const ConditionKey& conditionKeys, bool condition,
		const LogEvent& event) {
		ALOGW("Not used in duration tracker.");
		}

		void DurationMetricProducer::onMatchedLogEventLocked(const size_t matcherIndex,
		const LogEvent& event) {
		uint64_t eventTimeNs = event.GetElapsedTimestampNs();
		if (eventTimeNs < mStartTimeNs) {
		return;
		}

		flushIfNeededLocked(event.GetElapsedTimestampNs());

		// Handles Stopall events.
		if (matcherIndex == mStopAllIndex) {
		for (auto& pair : mCurrentSlicedDurationTrackerMap) {
		for (auto& whatIt : mCurrentSlicedDurationTrackerMap) {
		for (auto& pair : whatIt.second) {
		pair.second->noteStopAll(event.GetElapsedTimestampNs());
		}
		}
		return;
		}

		if (mCurrentSlicedDurationTrackerMap.find(eventKey) == mCurrentSlicedDurationTrackerMap.end()) {
		if (hitGuardRailLocked(eventKey)) {
		vector<HashableDimensionKey> dimensionInWhatValues;
		if (!mDimensionsInWhat.empty()) {
		filterValues(mDimensionsInWhat, event.getValues(), &dimensionInWhatValues);
		} else {
		dimensionInWhatValues.push_back(DEFAULT_DIMENSION_KEY);
		}

		// Handles Stop events.
		if (matcherIndex == mStopIndex) {
		if (mUseWhatDimensionAsInternalDimension) {
		for (const HashableDimensionKey& whatKey : dimensionInWhatValues) {
		auto whatIt = mCurrentSlicedDurationTrackerMap.find(whatKey);
		if (whatIt != mCurrentSlicedDurationTrackerMap.end()) {
		for (const auto& condIt : whatIt->second) {
		condIt.second->noteStop(whatKey, event.GetElapsedTimestampNs(), false);
		}
		}
		}
		return;
		}
		mCurrentSlicedDurationTrackerMap[eventKey] = createDurationTracker(eventKey);

		std::vector<HashableDimensionKey> internalDimensionKeys;
		filterValues(mInternalDimensions, event.getValues(), &internalDimensionKeys);
		if (internalDimensionKeys.empty()) {
		internalDimensionKeys.push_back(DEFAULT_DIMENSION_KEY);
		}
		for (const HashableDimensionKey& whatDimension : dimensionInWhatValues) {
		auto whatIt = mCurrentSlicedDurationTrackerMap.find(whatDimension);
		if (whatIt != mCurrentSlicedDurationTrackerMap.end()) {
		for (const auto& condIt : whatIt->second) {
		for (const auto& internalDimensionKey : internalDimensionKeys) {
		condIt.second->noteStop(
		internalDimensionKey, event.GetElapsedTimestampNs(), false);
		}
		}
		}
		}
		return;
		}

		auto it = mCurrentSlicedDurationTrackerMap.find(eventKey);
		bool condition;
		ConditionKey conditionKey;
		std::unordered_set<HashableDimensionKey> dimensionKeysInCondition;
		if (mConditionSliced) {
		for (const auto& link : mMetric2ConditionLinks) {
		getDimensionForCondition(event.getValues(), link, &conditionKey[link.conditionId]);
		}

		std::vector<HashableDimensionKey> values;
		filterValues(mInternalDimensions, event.getValues(), &values);
		if (values.empty()) {
		if (matcherIndex == mStartIndex) {
		it->second->noteStart(DEFAULT_DIMENSION_KEY, condition,
		event.GetElapsedTimestampNs(), conditionKeys);
		} else if (matcherIndex == mStopIndex) {
		it->second->noteStop(DEFAULT_DIMENSION_KEY, event.GetElapsedTimestampNs(), false);
		auto conditionState =
		mWizard->query(mConditionTrackerIndex, conditionKey, mDimensionsInCondition,
		&dimensionKeysInCondition);
		condition = (conditionState == ConditionState::kTrue);
		if (mDimensionsInCondition.empty() && condition) {
		dimensionKeysInCondition.insert(DEFAULT_DIMENSION_KEY);
		}
		} else {
		for (const auto& value : values) {
		if (matcherIndex == mStartIndex) {
		it->second->noteStart(value, condition, event.GetElapsedTimestampNs(), conditionKeys);
		} else if (matcherIndex == mStopIndex) {
		it->second->noteStop(value, event.GetElapsedTimestampNs(), false);
		}
		condition = mCondition;
		if (condition) {
		dimensionKeysInCondition.insert(DEFAULT_DIMENSION_KEY);
		}
		}

		for (const auto& whatDimension : dimensionInWhatValues) {
		auto whatIt = mCurrentSlicedDurationTrackerMap.find(whatDimension);
		// If the what dimension is already there, we should update all the trackers even
		// the condition is false.
		if (whatIt != mCurrentSlicedDurationTrackerMap.end()) {
		for (const auto& condIt : whatIt->second) {
		const bool cond = dimensionKeysInCondition.find(condIt.first) !=
		dimensionKeysInCondition.end();
		handleStartEvent(MetricDimensionKey(whatDimension, condIt.first),
		conditionKey, cond, event);
		}
		} else {
		// If it is a new what dimension key, we need to handle the start events for all current
		// condition dimensions.
		for (const auto& conditionDimension : dimensionKeysInCondition) {
		handleStartEvent(MetricDimensionKey(whatDimension, conditionDimension),
		conditionKey, condition, event);
		}
		}
		if (dimensionKeysInCondition.empty()) {
		handleStartEvent(MetricDimensionKey(whatDimension, DEFAULT_DIMENSION_KEY),
		conditionKey, condition, event);
		}
		}
		}


		size_t DurationMetricProducer::byteSizeLocked() const {
		size_t totalSize = 0;

cmds/statsd/src/metrics/DurationMetricProducer.h

+10 −2

Original line number	Diff line number	Diff line
		@@ -50,12 +50,16 @@ public:
		const sp<AlarmMonitor>& anomalyAlarmMonitor) override;

		protected:
		void onMatchedLogEventLocked(const size_t matcherIndex, const LogEvent& event) override;
		void onMatchedLogEventInternalLocked(
		const size_t matcherIndex, const MetricDimensionKey& eventKey,
		const ConditionKey& conditionKeys, bool condition,
		const LogEvent& event) override;

		private:
		void handleStartEvent(const MetricDimensionKey& eventKey, const ConditionKey& conditionKeys,
		bool condition, const LogEvent& event);

		void onDumpReportLocked(const uint64_t dumpTimeNs,
		android::util::ProtoOutputStream* protoOutput) override;

		@@ -92,12 +96,16 @@ private:
		// The dimension from the atom predicate. e.g., uid, wakelock name.
		vector<Matcher> mInternalDimensions;

		// This boolean is true iff When mInternalDimensions == mDimensionsInWhat
		bool mUseWhatDimensionAsInternalDimension;

		// Save the past buckets and we can clear when the StatsLogReport is dumped.
		// TODO: Add a lock to mPastBuckets.
		std::unordered_map<MetricDimensionKey, std::vector<DurationBucket>> mPastBuckets;

		// The current bucket.
		std::unordered_map<MetricDimensionKey, std::unique_ptr<DurationTracker>>
		// The duration trackers in the current bucket.
		std::unordered_map<HashableDimensionKey,
		std::unordered_map<HashableDimensionKey, std::unique_ptr<DurationTracker>>>
		mCurrentSlicedDurationTrackerMap;

		// Helper function to create a duration tracker given the metric aggregation type.