Merge "Log bucket drop reasons for ValueMetric and GaugeMetric" (9e44c0a9) · Commits · e / os / android_frameworks_base

cmds/statsd/src/guardrail/StatsdStats.h

+2 −0

Original line number	Diff line number	Diff line
		@@ -181,6 +181,8 @@ public:

		static const int64_t kInt64Max = 0x7fffffffffffffffLL;

		static const int32_t kMaxLoggedBucketDropEvents = 10;

		/**
		* Report a new config has been received and report the static stats about the config.
		*

cmds/statsd/src/metrics/GaugeMetricProducer.cpp

+29 −6

Original line number	Diff line number	Diff line
		@@ -52,8 +52,13 @@ const int FIELD_ID_IS_ACTIVE = 14;
		// for GaugeMetricDataWrapper
		const int FIELD_ID_DATA = 1;
		const int FIELD_ID_SKIPPED = 2;
		// for SkippedBuckets
		const int FIELD_ID_SKIPPED_START_MILLIS = 3;
		const int FIELD_ID_SKIPPED_END_MILLIS = 4;
		const int FIELD_ID_SKIPPED_DROP_EVENT = 5;
		// for DumpEvent Proto
		const int FIELD_ID_BUCKET_DROP_REASON = 1;
		const int FIELD_ID_DROP_TIME = 2;
		// for GaugeMetricData
		const int FIELD_ID_DIMENSION_IN_WHAT = 1;
		const int FIELD_ID_BUCKET_INFO = 3;
		@@ -193,7 +198,7 @@ void GaugeMetricProducer::onDumpReportLocked(const int64_t dumpTimeNs,
		protoOutput->write(FIELD_TYPE_INT64 \| FIELD_ID_ID, (long long)mMetricId);
		protoOutput->write(FIELD_TYPE_BOOL \| FIELD_ID_IS_ACTIVE, isActiveLocked());

		if (mPastBuckets.empty()) {
		if (mPastBuckets.empty() && mSkippedBuckets.empty()) {
		return;
		}

		@@ -212,13 +217,21 @@ void GaugeMetricProducer::onDumpReportLocked(const int64_t dumpTimeNs,

		uint64_t protoToken = protoOutput->start(FIELD_TYPE_MESSAGE \| FIELD_ID_GAUGE_METRICS);

		for (const auto& pair : mSkippedBuckets) {
		for (const auto& skippedBucket : mSkippedBuckets) {
		uint64_t wrapperToken =
		protoOutput->start(FIELD_TYPE_MESSAGE \| FIELD_COUNT_REPEATED \| FIELD_ID_SKIPPED);
		protoOutput->write(FIELD_TYPE_INT64 \| FIELD_ID_SKIPPED_START_MILLIS,
		(long long)(NanoToMillis(pair.first)));
		(long long)(NanoToMillis(skippedBucket.bucketStartTimeNs)));
		protoOutput->write(FIELD_TYPE_INT64 \| FIELD_ID_SKIPPED_END_MILLIS,
		(long long)(NanoToMillis(pair.second)));
		(long long)(NanoToMillis(skippedBucket.bucketEndTimeNs)));

		for (const auto& dropEvent : skippedBucket.dropEvents) {
		uint64_t dropEventToken = protoOutput->start(FIELD_TYPE_MESSAGE \| FIELD_COUNT_REPEATED \|
		FIELD_ID_SKIPPED_DROP_EVENT);
		protoOutput->write(FIELD_TYPE_INT32 \| FIELD_ID_BUCKET_DROP_REASON, dropEvent.reason);
		protoOutput->write(FIELD_TYPE_INT64 \| FIELD_ID_DROP_TIME, (long long) (NanoToMillis(dropEvent.dropTimeNs)));
		protoOutput->end(dropEventToken);
		}
		protoOutput->end(wrapperToken);
		}

		@@ -545,7 +558,10 @@ void GaugeMetricProducer::flushCurrentBucketLocked(const int64_t& eventTimeNs,
		info.mBucketEndNs = fullBucketEndTimeNs;
		}

		if (info.mBucketEndNs - mCurrentBucketStartTimeNs >= mMinBucketSizeNs) {
		// Add bucket to mPastBuckets if bucket is large enough.
		// Otherwise, drop the bucket data and add bucket metadata to mSkippedBuckets.
		bool isBucketLargeEnough = info.mBucketEndNs - mCurrentBucketStartTimeNs >= mMinBucketSizeNs;
		if (isBucketLargeEnough) {
		for (const auto& slice : *mCurrentSlicedBucket) {
		info.mGaugeAtoms = slice.second;
		auto& bucketList = mPastBuckets[slice.first];
		@@ -554,7 +570,13 @@ void GaugeMetricProducer::flushCurrentBucketLocked(const int64_t& eventTimeNs,
		slice.first.toString().c_str());
		}
		} else {
		mSkippedBuckets.emplace_back(info.mBucketStartNs, info.mBucketEndNs);
		mCurrentSkippedBucket.bucketStartTimeNs = mCurrentBucketStartTimeNs;
		mCurrentSkippedBucket.bucketEndTimeNs = eventTimeNs;
		if (!maxDropEventsReached()) {
		mCurrentSkippedBucket.dropEvents.emplace_back(
		buildDropEvent(eventTimeNs, BucketDropReason::BUCKET_TOO_SMALL));
		}
		mSkippedBuckets.emplace_back(mCurrentSkippedBucket);
		}

		// If we have anomaly trackers, we need to update the partial bucket values.
		@@ -573,6 +595,7 @@ void GaugeMetricProducer::flushCurrentBucketLocked(const int64_t& eventTimeNs,
		StatsdStats::getInstance().noteBucketCount(mMetricId);
		mCurrentSlicedBucket = std::make_shared<DimToGaugeAtomsMap>();
		mCurrentBucketStartTimeNs = nextBucketStartTimeNs;
		mCurrentSkippedBucket.reset();
		}

		size_t GaugeMetricProducer::byteSizeLocked() const {

cmds/statsd/src/metrics/GaugeMetricProducer.h

+0 −3

Original line number	Diff line number	Diff line
		@@ -158,9 +158,6 @@ private:
		// this slice (ie, for partial buckets, we use the last partial bucket in this full bucket).
		std::shared_ptr<DimToValMap> mCurrentSlicedBucketForAnomaly;

		// Pairs of (elapsed start, elapsed end) denoting buckets that were skipped.
		std::list<std::pair<int64_t, int64_t>> mSkippedBuckets;

		const int64_t mMinBucketSizeNs;

		// Translate Atom based bucket to single numeric value bucket for anomaly and updates the map

cmds/statsd/src/metrics/MetricProducer.cpp

+12 −0

Original line number	Diff line number	Diff line
		@@ -19,6 +19,7 @@

		#include "MetricProducer.h"

		#include "../guardrail/StatsdStats.h"
		#include "state/StateTracker.h"

		using android::util::FIELD_COUNT_REPEATED;
		@@ -289,6 +290,17 @@ void MetricProducer::getMappedStateValue(const int32_t atomId, const HashableDim
		}
		}

		DropEvent MetricProducer::buildDropEvent(const int64_t dropTimeNs, const BucketDropReason reason) {
		DropEvent event;
		event.reason = reason;
		event.dropTimeNs = dropTimeNs;
		return event;
		}

		bool MetricProducer::maxDropEventsReached() {
		return mCurrentSkippedBucket.dropEvents.size() >= StatsdStats::kMaxLoggedBucketDropEvents;
		}

		} // namespace statsd
		} // namespace os
		} // namespace android

cmds/statsd/src/metrics/MetricProducer.h

+48 −0

Original line number	Diff line number	Diff line
		@@ -70,6 +70,22 @@ enum DumpLatency {
		NO_TIME_CONSTRAINTS = 2
		};

		// Keep this in sync with BucketDropReason enum in stats_log.proto
		enum BucketDropReason {
		// For ValueMetric, a bucket is dropped during a dump report request iff
		// current bucket should be included, a pull is needed (pulled metric and
		// condition is true), and we are under fast time constraints.
		DUMP_REPORT_REQUESTED = 1,
		EVENT_IN_WRONG_BUCKET = 2,
		CONDITION_UNKNOWN = 3,
		PULL_FAILED = 4,
		PULL_DELAYED = 5,
		DIMENSION_GUARDRAIL_REACHED = 6,
		MULTIPLE_BUCKETS_SKIPPED = 7,
		// Not an invalid bucket case, but the bucket is dropped.
		BUCKET_TOO_SMALL = 8
		};

		struct Activation {
		Activation(const ActivationType& activationType, const int64_t ttlNs)
		: ttl_ns(ttlNs),
		@@ -83,6 +99,28 @@ struct Activation {
		const ActivationType activationType;
		};

		struct DropEvent {
		// Reason for dropping the bucket and/or marking the bucket invalid.
		BucketDropReason reason;
		// The timestamp of the drop event.
		int64_t dropTimeNs;
		};

		struct SkippedBucket {
		// Start time of the dropped bucket.
		int64_t bucketStartTimeNs;
		// End time of the dropped bucket.
		int64_t bucketEndTimeNs;
		// List of events that invalidated this bucket.
		std::vector<DropEvent> dropEvents;

		void reset() {
		bucketStartTimeNs = 0;
		bucketEndTimeNs = 0;
		dropEvents.clear();
		}
		};

		// A MetricProducer is responsible for compute one single metrics, creating stats log report, and
		// writing the report to dropbox. MetricProducers should respond to package changes as required in
		// PackageInfoListener, but if none of the metrics are slicing by package name, then the update can
		@@ -342,6 +380,12 @@ protected:
		void getMappedStateValue(const int32_t atomId, const HashableDimensionKey& queryKey,
		FieldValue* value);

		DropEvent buildDropEvent(const int64_t dropTimeNs, const BucketDropReason reason);

		// Returns true if the number of drop events in the current bucket has
		// exceeded the maximum number allowed, which is currently capped at 10.
		bool maxDropEventsReached();

		const int64_t mMetricId;

		const ConfigKey mConfigKey;
		@@ -403,6 +447,10 @@ protected:
		// atom to fields in the "what" atom.
		std::vector<Metric2State> mMetric2StateLinks;

		SkippedBucket mCurrentSkippedBucket;
		// Buckets that were invalidated and had their data dropped.
		std::vector<SkippedBucket> mSkippedBuckets;

		FRIEND_TEST(CountMetricE2eTest, TestSlicedState);
		FRIEND_TEST(CountMetricE2eTest, TestSlicedStateWithMap);
		FRIEND_TEST(CountMetricE2eTest, TestMultipleSlicedStates);