Merge "Statsd CPU optimization." (e8cf6110) · Commits · e / os / android_frameworks_base

cmds/statsd/Android.mk

+2 −4

Original line number	Diff line number	Diff line
		@@ -19,11 +19,9 @@ statsd_common_src := \
		../../core/java/android/os/IStatsManager.aidl \
		src/stats_log.proto \
		src/statsd_config.proto \
		src/statsd_internal.proto \
		src/atoms.proto \
		src/field_util.cpp \
		src/FieldValue.cpp \
		src/stats_log_util.cpp \
		src/dimension.cpp \
		src/anomaly/AnomalyMonitor.cpp \
		src/anomaly/AnomalyTracker.cpp \
		src/anomaly/DurationAnomalyTracker.cpp \
		@@ -172,7 +170,6 @@ LOCAL_CFLAGS += \

		LOCAL_SRC_FILES := \
		$(statsd_common_src) \
		tests/dimension_test.cpp \
		tests/AnomalyMonitor_test.cpp \
		tests/anomaly/AnomalyTracker_test.cpp \
		tests/ConfigManager_test.cpp \
		@@ -184,6 +181,7 @@ LOCAL_SRC_FILES := \
		tests/MetricsManager_test.cpp \
		tests/StatsLogProcessor_test.cpp \
		tests/UidMap_test.cpp \
		tests/FieldValue_test.cpp \
		tests/condition/CombinationConditionTracker_test.cpp \
		tests/condition/SimpleConditionTracker_test.cpp \
		tests/metrics/OringDurationTracker_test.cpp \

cmds/statsd/src/FieldValue.cpp

0 → 100644

+186 −0

Original line number	Diff line number	Diff line
		/*
		* Copyright (C) 2018 The Android Open Source Project
		*
		* Licensed under the Apache License, Version 2.0 (the "License");
		* you may not use this file except in compliance with the License.
		* You may obtain a copy of the License at
		*
		* http://www.apache.org/licenses/LICENSE-2.0
		*
		* Unless required by applicable law or agreed to in writing, software
		* distributed under the License is distributed on an "AS IS" BASIS,
		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
		* See the License for the specific language governing permissions and
		* limitations under the License.
		*/

		#define DEBUG false
		#include "Log.h"
		#include "FieldValue.h"
		#include "HashableDimensionKey.h"

		namespace android {
		namespace os {
		namespace statsd {

		bool Field::matches(const Matcher& matcher) const {
		if (mTag != matcher.mMatcher.getTag()) {
		return false;
		}
		if ((mField & matcher.mMask) == matcher.mMatcher.getField()) {
		return true;
		}

		return false;
		};

		void translateFieldMatcher(int tag, const FieldMatcher& matcher, int depth, int* pos, int* mask,
		std::vector<Matcher>* output) {
		if (depth > kMaxLogDepth) {
		ALOGE("depth > 2");
		return;
		}

		pos[depth] = matcher.field();
		mask[depth] = 0x7f;

		if (matcher.has_position()) {
		depth++;
		if (depth > 2) {
		return;
		}
		switch (matcher.position()) {
		case Position::ANY:
		pos[depth] = 0;
		mask[depth] = 0;
		break;
		case Position::FIRST:
		pos[depth] = 1;
		mask[depth] = 0x7f;
		break;
		case Position::LAST:
		pos[depth] = 0x80;
		mask[depth] = 0x80;
		break;
		case Position::POSITION_UNKNOWN:
		pos[depth] = 0;
		mask[depth] = 0;
		break;
		}
		}

		if (matcher.child_size() == 0) {
		output->push_back(Matcher(Field(tag, pos, depth), encodeMatcherMask(mask, depth)));
		Matcher matcher = Matcher(Field(tag, pos, depth), encodeMatcherMask(mask, depth));
		} else {
		for (const auto& child : matcher.child()) {
		translateFieldMatcher(tag, child, depth + 1, pos, mask, output);
		}
		}
		}

		void translateFieldMatcher(const FieldMatcher& matcher, std::vector<Matcher>* output) {
		int pos[] = {1, 1, 1};
		int mask[] = {0x7f, 0x7f, 0x7f};
		int tag = matcher.field();
		for (const auto& child : matcher.child()) {
		translateFieldMatcher(tag, child, 0, pos, mask, output);
		}
		}

		bool isAttributionUidField(const FieldValue& value) {
		int field = value.mField.getField() & 0xff007f;
		if (field == 0x10001 && value.mValue.getType() == INT) {
		return true;
		}
		return false;
		}

		bool isAttributionUidField(const Field& field, const Value& value) {
		int f = field.getField() & 0xff007f;
		if (f == 0x10001 && value.getType() == INT) {
		return true;
		}
		return false;
		}

		Value::Value(const Value& from) {
		type = from.getType();
		switch (type) {
		case INT:
		int_value = from.int_value;
		break;
		case LONG:
		long_value = from.long_value;
		break;
		case FLOAT:
		float_value = from.float_value;
		break;
		case STRING:
		str_value = from.str_value;
		break;
		}
		}

		std::string Value::toString() const {
		switch (type) {
		case INT:
		return std::to_string(int_value) + "[I]";
		case LONG:
		return std::to_string(long_value) + "[L]";
		case FLOAT:
		return std::to_string(float_value) + "[F]";
		case STRING:
		return str_value + "[S]";
		}
		}

		bool Value::operator==(const Value& that) const {
		if (type != that.getType()) return false;

		switch (type) {
		case INT:
		return int_value == that.int_value;
		case LONG:
		return long_value == that.long_value;
		case FLOAT:
		return float_value == that.float_value;
		case STRING:
		return str_value == that.str_value;
		}
		}

		bool Value::operator!=(const Value& that) const {
		if (type != that.getType()) return true;
		switch (type) {
		case INT:
		return int_value != that.int_value;
		case LONG:
		return long_value != that.long_value;
		case FLOAT:
		return float_value != that.float_value;
		case STRING:
		return str_value != that.str_value;
		}
		}

		bool Value::operator<(const Value& that) const {
		if (type != that.getType()) return type < that.getType();

		switch (type) {
		case INT:
		return int_value < that.int_value;
		case LONG:
		return long_value < that.long_value;
		case FLOAT:
		return float_value < that.float_value;
		case STRING:
		return str_value < that.str_value;
		default:
		return false;
		}
		}

		} // namespace statsd
		} // namespace os
		} // namespace android
		No newline at end of file

cmds/statsd/src/FieldValue.h

0 → 100644

+338 −0

Original line number	Diff line number	Diff line
		/*
		* Copyright (C) 2018 The Android Open Source Project
		*
		* Licensed under the Apache License, Version 2.0 (the "License");
		* you may not use this file except in compliance with the License.
		* You may obtain a copy of the License at
		*
		* http://www.apache.org/licenses/LICENSE-2.0
		*
		* Unless required by applicable law or agreed to in writing, software
		* distributed under the License is distributed on an "AS IS" BASIS,
		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
		* See the License for the specific language governing permissions and
		* limitations under the License.
		*/
		#pragma once

		#include "frameworks/base/cmds/statsd/src/statsd_config.pb.h"

		namespace android {
		namespace os {
		namespace statsd {

		class HashableDimensionKey;
		struct Matcher;
		struct Field;
		struct FieldValue;

		const int32_t kAttributionField = 1;
		const int32_t kMaxLogDepth = 2;
		const int32_t kLastBitMask = 0x80;
		const int32_t kClearLastBitDeco = 0x7f;

		enum Type { INT, LONG, FLOAT, STRING };


		static int32_t getEncodedField(int32_t pos[], int32_t depth, bool includeDepth) {
		int32_t field = 0;
		for (int32_t i = 0; i <= depth; i++) {
		int32_t shiftBits = 8 * (kMaxLogDepth - i);
		field \|= (pos[i] << shiftBits);
		}

		if (includeDepth) {
		field \|= (depth << 24);
		}
		return field;
		}

		static int32_t encodeMatcherMask(int32_t mask[], int32_t depth) {
		return getEncodedField(mask, depth, false) \| 0xff000000;
		}

		// Get the encoded field for a leaf with a [field] number at depth 0;
		static int32_t getSimpleField(size_t field) {
		return ((int32_t)field << 8 * 2);
		}

		/**
		* Field is a wrapper class for 2 integers that represents the field of a log element in its Atom
		* proto.
		* [mTag]: the atom id.
		* [mField]: encoded path from the root (atom) to leaf.
		*
		* For example:
		* WakeLockStateChanged {
		* repeated AttributionNode = 1;
		* int state = 2;
		* string tag = 3;
		* }
		* Read from logd, the items are structured as below:
		* [[[1000, "tag"], [2000, "tag2"],], 2,"hello"]
		*
		* When we read through the list, we will encode each field in a 32bit integer.
		* 8bit segments \|--------\|--------\|--------\|--------\|
		* Depth field0 [L]field1 [L]field1
		*
		* The first 8 bits are the depth of the field. for example, the uid 1000 has depth 2.
		* The following 3 8-bit are for the item's position at each level.
		* The first bit of each 8bits field is reserved to mark if the item is the last item at that level
		* this is to make matching easier later.
		*
		* The above wakelock event is translated into FieldValue pairs.
		* 0x02010101->1000
		* 0x02010182->tag
		* 0x02018201->2000
		* 0x02018282->tag2
		* 0x00020000->2
		* 0x00030000->"hello"
		*
		* This encoding is the building block for the later operations.
		* Please see the definition for Matcher below to see how the matching is done.
		*/
		struct Field {
		private:
		int32_t mTag;
		int32_t mField;

		public:
		Field(int32_t tag, int32_t pos[], int32_t depth) : mTag(tag) {
		mField = getEncodedField(pos, depth, true);
		}

		Field(const Field& from) : mTag(from.getTag()), mField(from.getField()) {
		}

		Field(int32_t tag, int32_t field) : mTag(tag), mField(field){};

		inline void setField(int32_t field) {
		mField = field;
		}

		inline void setTag(int32_t tag) {
		mTag = tag;
		}

		inline void decorateLastPos(int32_t depth) {
		int32_t mask = kLastBitMask << 8 * (kMaxLogDepth - depth);
		mField \|= mask;
		}

		inline int32_t getTag() const {
		return mTag;
		}

		inline int32_t getDepth() const {
		return (mField >> 24);
		}

		inline int32_t getPath(int32_t depth) const {
		if (depth > 2 \|\| depth < 0) return 0;

		int32_t field = (mField & 0x00ffffff);
		int32_t mask = 0xffffffff;
		return (field & (mask << 8 * (kMaxLogDepth - depth)));
		}

		inline int32_t getPrefix(int32_t depth) const {
		if (depth == 0) return 0;
		return getPath(depth - 1);
		}

		inline int32_t getField() const {
		return mField;
		}

		inline int32_t getRawPosAtDepth(int32_t depth) const {
		int32_t field = (mField & 0x00ffffff);
		int32_t shift = 8 * (kMaxLogDepth - depth);
		int32_t mask = 0xff << shift;

		return (field & mask) >> shift;
		}

		inline int32_t getPosAtDepth(int32_t depth) const {
		return getRawPosAtDepth(depth) & kClearLastBitDeco;
		}

		// Check if the first bit of the 8-bit segment for depth is 1
		inline bool isLastPos(int32_t depth) const {
		int32_t field = (mField & 0x00ffffff);
		int32_t mask = kLastBitMask << 8 * (kMaxLogDepth - depth);
		return (field & mask) != 0;
		}

		// if the 8-bit segment is all 0's
		inline bool isAnyPosMatcher(int32_t depth) const {
		return getDepth() >= depth && getRawPosAtDepth(depth) == 0;
		}
		// if the 8bit is 0x80 (1000 0000)
		inline bool isLastPosMatcher(int32_t depth) const {
		return getDepth() >= depth && getRawPosAtDepth(depth) == kLastBitMask;
		}

		inline bool operator==(const Field& that) const {
		return mTag == that.getTag() && mField == that.getField();
		};

		inline bool operator!=(const Field& that) const {
		return mTag != that.getTag() \|\| mField != that.getField();
		};

		bool operator<(const Field& that) const {
		if (mTag != that.getTag()) {
		return mTag < that.getTag();
		}

		if (mField != that.getField()) {
		return mField < that.getField();
		}

		return false;
		}
		bool matches(const Matcher& that) const;
		};

		/**
		* Matcher represents a leaf matcher in the FieldMatcher in statsd_config.
		*
		* It contains all information needed to match one or more leaf node.
		* All information is encoded in a Field(2 ints) and a bit mask(1 int).
		*
		* For example, to match the first/any/last uid field in attribution chain in Atom 10,
		* we have the following FieldMatcher in statsd_config
		* FieldMatcher {
		* field:10
		* FieldMatcher {
		* field:1
		* position: any/last/first
		* FieldMatcher {
		* field:1
		* }
		* }
		* }
		*
		* We translate the FieldMatcher into a Field, and mask
		* First: [Matcher Field] 0x02010101 [Mask]0xffff7fff
		* Last: [Matcher Field] 0x02018001 [Mask]0xffff80ff
		* Any: [Matcher Field] 0x02010001 [Mask]0xffff00ff
		*
		* [To match a log Field with a Matcher] we apply the bit mask to the log Field and check if
		* the result is equal to the Matcher Field. That's a bit wise AND operation + check if 2 ints are
		* equal. Nothing can beat the performance of this matching algorithm.
		*
		* TODO: ADD EXAMPLE HERE.
		*/
		struct Matcher {
		Matcher(const Field& matcher, int32_t mask) : mMatcher(matcher), mMask(mask){};

		const Field mMatcher;
		const int32_t mMask;

		bool hasAnyPositionMatcher(int* prefix) const {
		if (mMatcher.getDepth() == 2 && mMatcher.getRawPosAtDepth(2) == 0) {
		(*prefix) = mMatcher.getPrefix(2);
		return true;
		}
		return false;
		}
		};

		/**
		* A wrapper for a union type to contain multiple types of values.
		*
		*/
		struct Value {
		Value(int32_t v) {
		int_value = v;
		type = INT;
		}

		Value(int64_t v) {
		long_value = v;
		type = LONG;
		}

		Value(float v) {
		float_value = v;
		type = FLOAT;
		}

		Value(const std::string& v) {
		str_value = v;
		type = STRING;
		}

		void setInt(int32_t v) {
		int_value = v;
		type = INT;
		}

		void setLong(int64_t v) {
		long_value = v;
		type = LONG;
		}

		union {
		int32_t int_value;
		int64_t long_value;
		float float_value;
		};
		std::string str_value;

		Type type;

		std::string toString() const;

		Type getType() const {
		return type;
		}

		Value(const Value& from);

		bool operator==(const Value& that) const;
		bool operator!=(const Value& that) const;

		bool operator<(const Value& that) const;

		private:
		Value(){};
		};

		/**
		* Represents a log item, or a dimension item (They are essentially the same).
		*/
		struct FieldValue {
		FieldValue(const Field& field, const Value& value) : mField(field), mValue(value) {
		}
		bool operator==(const FieldValue& that) const {
		return mField == that.mField && mValue == that.mValue;
		}
		bool operator!=(const FieldValue& that) const {
		return mField != that.mField \|\| mValue != that.mValue;
		}
		bool operator<(const FieldValue& that) const {
		if (mField != that.mField) {
		return mField < that.mField;
		}

		if (mValue != that.mValue) {
		return mValue < that.mValue;
		}

		return false;
		}

		Field mField;
		Value mValue;
		};

		bool isAttributionUidField(const FieldValue& value);

		void translateFieldMatcher(const FieldMatcher& matcher, std::vector<Matcher>* output);

		bool isAttributionUidField(const Field& field, const Value& value);
		} // namespace statsd
		} // namespace os
		} // namespace android

cmds/statsd/src/HashableDimensionKey.cpp

+205 −129

File changed.

Preview size limit exceeded, changes collapsed.

cmds/statsd/src/HashableDimensionKey.h

+66 −25

Original line number	Diff line number	Diff line
		@@ -17,44 +17,66 @@
		#pragma once

		#include <utils/JenkinsHash.h>
		#include "frameworks/base/cmds/statsd/src/stats_log.pb.h"
		#include <vector>
		#include "FieldValue.h"
		#include "android-base/stringprintf.h"
		#include "logd/LogEvent.h"

		namespace android {
		namespace os {
		namespace statsd {

		using android::base::StringPrintf;

		struct Metric2Condition {
		int64_t conditionId;
		std::vector<Matcher> metricFields;
		std::vector<Matcher> conditionFields;
		};

		class HashableDimensionKey {
		public:
		explicit HashableDimensionKey(const DimensionsValue& dimensionsValue)
		: mDimensionsValue(dimensionsValue){};
		explicit HashableDimensionKey(const std::vector<FieldValue>& values) {
		mValues = values;
		}

		HashableDimensionKey(){};

		HashableDimensionKey(const HashableDimensionKey& that)
		: mDimensionsValue(that.getDimensionsValue()){};

		HashableDimensionKey& operator=(const HashableDimensionKey& from) = default;
		HashableDimensionKey(const HashableDimensionKey& that) : mValues(that.getValues()){};

		std::string toString() const;
		inline void addValue(const FieldValue& value) {
		mValues.push_back(value);
		}

		inline const DimensionsValue& getDimensionsValue() const {
		return mDimensionsValue;
		inline const std::vector<FieldValue>& getValues() const {
		return mValues;
		}

		inline DimensionsValue* getMutableDimensionsValue() {
		return &mDimensionsValue;
		inline std::vector<FieldValue>* mutableValues() {
		return &mValues;
		}

		bool operator==(const HashableDimensionKey& that) const;
		inline FieldValue* mutableValue(size_t i) {
		if (i >= 0 && i < mValues.size()) {
		return &(mValues[i]);
		}
		return nullptr;
		}

		bool operator<(const HashableDimensionKey& that) const;
		std::string toString() const;

		inline const char* c_str() const {
		return toString().c_str();
		}

		bool operator==(const HashableDimensionKey& that) const;

		bool operator<(const HashableDimensionKey& that) const;

		bool contains(const HashableDimensionKey& that) const;

		private:
		DimensionsValue mDimensionsValue;
		std::vector<FieldValue> mValues;
		};

		class MetricDimensionKey {
		@@ -83,7 +105,7 @@ class MetricDimensionKey {
		}

		bool hasDimensionKeyInCondition() const {
		return mDimensionKeyInCondition.getDimensionsValue().has_field();
		return mDimensionKeyInCondition.getValues().size() > 0;
		}

		bool operator==(const MetricDimensionKey& that) const;
		@@ -98,11 +120,32 @@ class MetricDimensionKey {
		HashableDimensionKey mDimensionKeyInCondition;
		};

		bool compareDimensionsValue(const DimensionsValue& s1, const DimensionsValue& s2);
		android::hash_t hashDimension(const HashableDimensionKey& key);

		/**
		* Creating HashableDimensionKeys from FieldValues using matcher.
		*
		* This function may make modifications to the Field if the matcher has Position=LAST or ANY in
		* it. This is because: for example, when we create dimension from last uid in attribution chain,
		* In one event, uid 1000 is at position 5 and it's the last
		* In another event, uid 1000 is at position 6, and it's the last
		* these 2 events should be mapped to the same dimension. So we will remove the original position
		* from the dimension key for the uid field (by applying 0x80 bit mask).
		*/
		bool filterValues(const std::vector<Matcher>& matcherFields, const std::vector<FieldValue>& values,
		std::vector<HashableDimensionKey>* output);

		/**
		* Filter the values from FieldValues using the matchers.
		*
		* In contrast to the above function, this function will not do any modification to the original
		* data. Considering it as taking a snapshot on the atom event.
		*/
		void filterGaugeValues(const std::vector<Matcher>& matchers, const std::vector<FieldValue>& values,
		std::vector<FieldValue>* output);

		android::hash_t hashDimensionsValue(int64_t seed, const DimensionsValue& value);
		android::hash_t hashDimensionsValue(const DimensionsValue& value);
		android::hash_t hashMetricDimensionKey(int64_t see, const MetricDimensionKey& dimensionKey);
		void getDimensionForCondition(const LogEvent& event, Metric2Condition links,
		std::vector<HashableDimensionKey>* conditionDimension);

		} // namespace statsd
		} // namespace os
		@@ -116,17 +159,15 @@ using android::os::statsd::MetricDimensionKey;
		template <>
		struct hash<HashableDimensionKey> {
		std::size_t operator()(const HashableDimensionKey& key) const {
		return hashDimensionsValue(key.getDimensionsValue());
		return hashDimension(key);
		}
		};

		template <>
		struct hash<MetricDimensionKey> {
		std::size_t operator()(const MetricDimensionKey& key) const {
		android::hash_t hash = hashDimensionsValue(
		key.getDimensionKeyInWhat().getDimensionsValue());
		hash = android::JenkinsHashMix(hash,
		hashDimensionsValue(key.getDimensionKeyInCondition().getDimensionsValue()));
		android::hash_t hash = hashDimension(key.getDimensionKeyInWhat());
		hash = android::JenkinsHashMix(hash, hashDimension(key.getDimensionKeyInCondition()));
		return android::JenkinsHashWhiten(hash);
		}
		};