Merge "Adds guardrail for memory usage for statsd uid map." (0044cfd9) · Commits · e / os / android_frameworks_base

cmds/statsd/src/StatsLogProcessor.cpp

+1 −1

Original line number	Diff line number	Diff line
		@@ -220,7 +220,7 @@ void StatsLogProcessor::flushIfNecessary(uint64_t timestampNs,
		const unique_ptr<MetricsManager>& metricsManager) {
		std::lock_guard<std::mutex> lock(mBroadcastTimesMutex);

		size_t totalBytes = metricsManager->byteSize();
		size_t totalBytes = metricsManager->byteSize() + mUidMap->getBytesUsed();
		if (totalBytes > .9 * kMaxSerializedBytes) { // Send broadcast so that receivers can pull data.
		auto lastFlushNs = mLastBroadcastTimes.find(key);
		if (lastFlushNs != mLastBroadcastTimes.end()) {

cmds/statsd/src/StatsService.cpp

+1 −1

Original line number	Diff line number	Diff line
		@@ -480,7 +480,7 @@ status_t StatsService::cmd_print_stats(FILE* out, const Vector<String8>& args) {
		fprintf(out, "Config %s uses %zu bytes\n", key.ToString().c_str(),
		mProcessor->GetMetricsSize(key));
		}
		fprintf(out, "Detailed statsd stats in logcat...");
		fprintf(out, "Detailed statsd stats in logcat...\n");
		StatsdStats& statsdStats = StatsdStats::getInstance();
		bool reset = false;
		if (args.size() > 1) {

cmds/statsd/src/guardrail/StatsdStats.cpp

+31 −0

Original line number	Diff line number	Diff line
		@@ -45,6 +45,7 @@ const int FIELD_ID_MATCHER_STATS = 4;
		const int FIELD_ID_CONDITION_STATS = 5;
		const int FIELD_ID_METRIC_STATS = 6;
		const int FIELD_ID_ATOM_STATS = 7;
		const int FIELD_ID_UIDMAP_STATS = 8;

		const int FIELD_ID_MATCHER_STATS_NAME = 1;
		const int FIELD_ID_MATCHER_STATS_COUNT = 2;
		@@ -173,6 +174,27 @@ void StatsdStats::noteMetricsReportSent(const ConfigKey& key, int32_t timeSec) {
		it->second.add_dump_report_time_sec(timeSec);
		}

		void StatsdStats::noteUidMapDropped(int snapshots, int deltas) {
		lock_guard<std::mutex> lock(mLock);
		mUidMapStats.set_dropped_snapshots(mUidMapStats.dropped_snapshots() + snapshots);
		mUidMapStats.set_dropped_changes(mUidMapStats.dropped_changes() + deltas);
		}

		void StatsdStats::setUidMapSnapshots(int snapshots) {
		lock_guard<std::mutex> lock(mLock);
		mUidMapStats.set_snapshots(snapshots);
		}

		void StatsdStats::setUidMapChanges(int changes) {
		lock_guard<std::mutex> lock(mLock);
		mUidMapStats.set_changes(changes);
		}

		void StatsdStats::setCurrentUidMapMemory(int bytes) {
		lock_guard<std::mutex> lock(mLock);
		mUidMapStats.set_bytes_used(bytes);
		}

		void StatsdStats::noteConditionDimensionSize(const ConfigKey& key, const string& name, int size) {
		lock_guard<std::mutex> lock(mLock);
		// if name doesn't exist before, it will create the key with count 0.
		@@ -364,6 +386,15 @@ void StatsdStats::dumpStats(std::vector<uint8_t>* output, bool reset) {
		}
		}

		const int numBytes = mUidMapStats.ByteSize();
		vector<char> buffer(numBytes);
		mUidMapStats.SerializeToArray(&buffer[0], numBytes);
		proto.write(FIELD_TYPE_MESSAGE \| FIELD_ID_UIDMAP_STATS, &buffer[0], buffer.size());
		VLOG("UID map stats: bytes=%d, snapshots=%d, changes=%d, snapshots lost=%d, changes "
		"lost=%d",
		mUidMapStats.bytes_used(), mUidMapStats.snapshots(), mUidMapStats.changes(),
		mUidMapStats.dropped_snapshots(), mUidMapStats.dropped_changes());

		output->clear();
		size_t bufferSize = proto.size();
		output->resize(bufferSize);

cmds/statsd/src/guardrail/StatsdStats.h

+19 −0

Original line number	Diff line number	Diff line
		@@ -45,6 +45,10 @@ public:

		const static int kMaxTimestampCount = 20;

		// Cap the UID map's memory usage to this. This should be fairly high since the UID information
		// is critical for understanding the metrics.
		const static size_t kMaxBytesUsedUidMap = 50 * 1024;

		/**
		* Report a new config has been received and report the static stats about the config.
		*
		@@ -112,6 +116,18 @@ public:
		*/
		void noteAtomLogged(int atomId, int32_t timeSec);

		/**
		* Records the number of snapshot and delta entries that are being dropped from the uid map.
		*/
		void noteUidMapDropped(int snapshots, int deltas);

		/**
		* Updates the number of snapshots currently stored in the uid map.
		*/
		void setUidMapSnapshots(int snapshots);
		void setUidMapChanges(int changes);
		void setCurrentUidMapMemory(int bytes);

		/**
		* Reset the historical stats. Including all stats in icebox, and the tracked stats about
		* metrics, matchers, and atoms. The active configs will be kept and StatsdStats will continue
		@@ -133,6 +149,9 @@ private:

		int32_t mStartTimeSec;

		// Track the number of dropped entries used by the uid map.
		StatsdStatsReport_UidMapStats mUidMapStats;

		// The stats about the configs that are still in use.
		std::map<const ConfigKey, StatsdStatsReport_ConfigStats> mConfigStats;

cmds/statsd/src/packages/UidMap.cpp

+76 −4

Original line number	Diff line number	Diff line
		@@ -13,11 +13,14 @@
		* See the License for the specific language governing permissions and
		* limitations under the License.
		*/

		#define DEBUG true // STOPSHIP if true
		#include "Log.h"

		#include "guardrail/StatsdStats.h"
		#include "packages/UidMap.h"

		#include <android/os/IStatsCompanionService.h>
		#include <binder/IServiceManager.h>
		#include <utils/Errors.h>

		using namespace android;
		@@ -26,6 +29,11 @@ namespace android {
		namespace os {
		namespace statsd {

		UidMap::UidMap() : mBytesUsed(0) {
		}
		UidMap::~UidMap() {
		}

		bool UidMap::hasApp(int uid, const string& packageName) const {
		lock_guard<mutex> lock(mMutex);

		@@ -73,6 +81,10 @@ void UidMap::updateMap(const int64_t& timestamp, const vector<int32_t>& uid,
		t->set_version(int(versionCode[j]));
		t->set_uid(uid[j]);
		}
		mBytesUsed += snapshot->ByteSize();
		StatsdStats::getInstance().setCurrentUidMapMemory(mBytesUsed);
		StatsdStats::getInstance().setUidMapSnapshots(mOutput.snapshots_size());
		ensureBytesUsedBelowLimit();
		}

		void UidMap::updateApp(const String16& app_16, const int32_t& uid, const int32_t& versionCode) {
		@@ -96,6 +108,10 @@ void UidMap::updateApp(const int64_t& timestamp, const String16& app_16, const i
		log->set_app(app);
		log->set_uid(uid);
		log->set_version(versionCode);
		mBytesUsed += log->ByteSize();
		StatsdStats::getInstance().setCurrentUidMapMemory(mBytesUsed);
		StatsdStats::getInstance().setUidMapChanges(mOutput.changes_size());
		ensureBytesUsedBelowLimit();

		auto range = mMap.equal_range(int(uid));
		for (auto it = range.first; it != range.second; ++it) {
		@@ -103,7 +119,7 @@ void UidMap::updateApp(const int64_t& timestamp, const String16& app_16, const i
		it->second.versionCode = int(versionCode);
		return;
		}
		ALOGD("updateApp failed to find the app %s with uid %i to update", app.c_str(), uid);
		VLOG("updateApp failed to find the app %s with uid %i to update", app.c_str(), uid);
		return;
		}

		@@ -111,6 +127,28 @@ void UidMap::updateApp(const int64_t& timestamp, const String16& app_16, const i
		mMap.insert(make_pair(uid, AppData(app, int(versionCode))));
		}

		void UidMap::ensureBytesUsedBelowLimit() {
		size_t limit;
		if (maxBytesOverride <= 0) {
		limit = StatsdStats::kMaxBytesUsedUidMap;
		} else {
		limit = maxBytesOverride;
		}
		while (mBytesUsed > limit) {
		VLOG("Bytes used %zu is above limit %zu, need to delete something", mBytesUsed, limit);
		if (mOutput.snapshots_size() > 0) {
		auto snapshots = mOutput.mutable_snapshots();
		snapshots->erase(snapshots->begin()); // Remove first snapshot.
		StatsdStats::getInstance().noteUidMapDropped(1, 0);
		} else if (mOutput.changes_size() > 0) {
		auto changes = mOutput.mutable_changes();
		changes->DeleteSubrange(0, 1);
		StatsdStats::getInstance().noteUidMapDropped(0, 1);
		}
		mBytesUsed = mOutput.ByteSize();
		}
		}

		void UidMap::removeApp(const String16& app_16, const int32_t& uid) {
		removeApp(time(nullptr) * NS_PER_SEC, app_16, uid);
		}
		@@ -128,6 +166,10 @@ void UidMap::removeApp(const int64_t& timestamp, const String16& app_16, const i
		log->set_timestamp_nanos(timestamp);
		log->set_app(app);
		log->set_uid(uid);
		mBytesUsed += log->ByteSize();
		StatsdStats::getInstance().setCurrentUidMapMemory(mBytesUsed);
		StatsdStats::getInstance().setUidMapChanges(mOutput.changes_size());
		ensureBytesUsedBelowLimit();

		auto range = mMap.equal_range(int(uid));
		for (auto it = range.first; it != range.second; ++it) {
		@@ -136,7 +178,7 @@ void UidMap::removeApp(const int64_t& timestamp, const String16& app_16, const i
		return;
		}
		}
		ALOGD("removeApp failed to find the app %s with uid %i to remove", app.c_str(), uid);
		VLOG("removeApp failed to find the app %s with uid %i to remove", app.c_str(), uid);
		return;
		}

		@@ -177,7 +219,6 @@ int UidMap::getParentUidOrSelf(int uid) {

		void UidMap::clearOutput() {
		mOutput.Clear();

		// Re-initialize the initial state for the outputs. This results in extra data being uploaded
		// but helps ensure we can re-construct the UID->app name, versionCode mapping in server.
		auto snapshot = mOutput.add_snapshots();
		@@ -187,6 +228,12 @@ void UidMap::clearOutput() {
		t->set_version(it.second.versionCode);
		t->set_uid(it.first);
		}

		// Also update the guardrail trackers.
		StatsdStats::getInstance().setUidMapChanges(0);
		StatsdStats::getInstance().setUidMapSnapshots(1);
		mBytesUsed = snapshot->ByteSize();
		StatsdStats::getInstance().setCurrentUidMapMemory(mBytesUsed);
		}

		int64_t UidMap::getMinimumTimestampNs() {
		@@ -201,6 +248,10 @@ int64_t UidMap::getMinimumTimestampNs() {
		return m;
		}

		size_t UidMap::getBytesUsed() {
		return mBytesUsed;
		}

		UidMapping UidMap::getOutput(const ConfigKey& key) {
		return getOutput(time(nullptr) * NS_PER_SEC, key);
		}
		@@ -236,6 +287,10 @@ UidMapping UidMap::getOutput(const int64_t& timestamp, const ConfigKey& key) {
		}
		}
		}
		mBytesUsed = mOutput.ByteSize(); // Compute actual size after potential deletions.
		StatsdStats::getInstance().setCurrentUidMapMemory(mBytesUsed);
		StatsdStats::getInstance().setUidMapChanges(mOutput.changes_size());
		StatsdStats::getInstance().setUidMapSnapshots(mOutput.snapshots_size());
		return ret;
		}

		@@ -250,6 +305,23 @@ void UidMap::printUidMap(FILE* out) {

		void UidMap::OnConfigUpdated(const ConfigKey& key) {
		mLastUpdatePerConfigKey[key] = -1;

		// Ensure there is at least one snapshot available since this configuration also needs to know
		// what all the uid's represent.
		if (mOutput.snapshots_size() == 0) {
		sp<IStatsCompanionService> statsCompanion = nullptr;
		// Get statscompanion service from service manager
		const sp<IServiceManager> sm(defaultServiceManager());
		if (sm != nullptr) {
		const String16 name("statscompanion");
		statsCompanion = interface_cast<IStatsCompanionService>(sm->checkService(name));
		if (statsCompanion == nullptr) {
		ALOGW("statscompanion service unavailable!");
		return;
		}
		statsCompanion->triggerUidSnapshot();
		}
		}
		}

		void UidMap::OnConfigRemoved(const ConfigKey& key) {