Merge "Fix recovery of stats data from previous input while using ProtoOutputStream" into pi-dev (e51af374) · Commits · e / os / android_frameworks_base

cmds/statsd/src/StatsLogProcessor.cpp

+50 −32

Original line number	Diff line number	Diff line
		@@ -239,14 +239,13 @@ void StatsLogProcessor::dumpStates(FILE* out, bool verbose) {
		}
		}

		/*
		* onDumpReport dumps serialized ConfigMetricsReportList into outData.
		*/
		void StatsLogProcessor::onDumpReport(const ConfigKey& key, const uint64_t dumpTimeStampNs,
		vector<uint8_t>* outData) {
		std::lock_guard<std::mutex> lock(mMetricsMutex);
		onDumpReportLocked(key, dumpTimeStampNs, outData);
		}

		void StatsLogProcessor::onDumpReportLocked(const ConfigKey& key, const uint64_t dumpTimeStampNs,
		vector<uint8_t>* outData) {
		auto it = mMetricsManagers.find(key);
		if (it == mMetricsManagers.end()) {
		ALOGW("Config source %s does not exist", key.ToString().c_str());
		@@ -269,35 +268,14 @@ void StatsLogProcessor::onDumpReportLocked(const ConfigKey& key, const uint64_t
		// Start of ConfigMetricsReport (reports).
		uint64_t reportsToken =
		proto.start(FIELD_TYPE_MESSAGE \| FIELD_COUNT_REPEATED \| FIELD_ID_REPORTS);

		int64_t lastReportTimeNs = it->second->getLastReportTimeNs();
		int64_t lastReportWallClockNs = it->second->getLastReportWallClockNs();

		// First, fill in ConfigMetricsReport using current data on memory, which
		// starts from filling in StatsLogReport's.
		it->second->onDumpReport(dumpTimeStampNs, &proto);

		// Fill in UidMap.
		vector<uint8_t> uidMap;
		mUidMap->getOutput(key, &uidMap);
		proto.write(FIELD_TYPE_MESSAGE \| FIELD_ID_UID_MAP, uidMap.data());

		// Fill in the timestamps.
		proto.write(FIELD_TYPE_INT64 \| FIELD_ID_LAST_REPORT_ELAPSED_NANOS,
		(long long)lastReportTimeNs);
		proto.write(FIELD_TYPE_INT64 \| FIELD_ID_CURRENT_REPORT_ELAPSED_NANOS,
		(long long)dumpTimeStampNs);
		proto.write(FIELD_TYPE_INT64 \| FIELD_ID_LAST_REPORT_WALL_CLOCK_NANOS,
		(long long)lastReportWallClockNs);
		proto.write(FIELD_TYPE_INT64 \| FIELD_ID_CURRENT_REPORT_WALL_CLOCK_NANOS,
		(long long)getWallClockNs());

		// End of ConfigMetricsReport (reports).
		onConfigMetricsReportLocked(key, dumpTimeStampNs, &proto);
		proto.end(reportsToken);
		// End of ConfigMetricsReport (reports).


		// Then, check stats-data directory to see there's any file containing
		// ConfigMetricsReport from previous shutdowns to concatenate to reports.
		StorageManager::appendConfigMetricsReport(proto);
		StorageManager::appendConfigMetricsReport(key, &proto);

		if (outData != nullptr) {
		outData->clear();
		@@ -315,6 +293,40 @@ void StatsLogProcessor::onDumpReportLocked(const ConfigKey& key, const uint64_t
		StatsdStats::getInstance().noteMetricsReportSent(key);
		}

		/*
		* onConfigMetricsReportLocked dumps serialized ConfigMetricsReport into outData.
		*/
		void StatsLogProcessor::onConfigMetricsReportLocked(const ConfigKey& key,
		const uint64_t dumpTimeStampNs,
		ProtoOutputStream* proto) {
		// We already checked whether key exists in mMetricsManagers in
		// WriteDataToDisk.
		auto it = mMetricsManagers.find(key);
		int64_t lastReportTimeNs = it->second->getLastReportTimeNs();
		int64_t lastReportWallClockNs = it->second->getLastReportWallClockNs();

		// First, fill in ConfigMetricsReport using current data on memory, which
		// starts from filling in StatsLogReport's.
		it->second->onDumpReport(dumpTimeStampNs, proto);

		// Fill in UidMap.
		uint64_t uidMapToken = proto->start(FIELD_TYPE_MESSAGE \| FIELD_ID_UID_MAP);
		mUidMap->appendUidMap(key, proto);
		proto->end(uidMapToken);

		// Fill in the timestamps.
		proto->write(FIELD_TYPE_INT64 \| FIELD_ID_LAST_REPORT_ELAPSED_NANOS,
		(long long)lastReportTimeNs);
		proto->write(FIELD_TYPE_INT64 \| FIELD_ID_CURRENT_REPORT_ELAPSED_NANOS,
		(long long)dumpTimeStampNs);
		proto->write(FIELD_TYPE_INT64 \| FIELD_ID_LAST_REPORT_WALL_CLOCK_NANOS,
		(long long)lastReportWallClockNs);
		proto->write(FIELD_TYPE_INT64 \| FIELD_ID_CURRENT_REPORT_WALL_CLOCK_NANOS,
		(long long)getWallClockNs());


		}

		void StatsLogProcessor::OnConfigRemoved(const ConfigKey& key) {
		std::lock_guard<std::mutex> lock(mMetricsMutex);
		auto it = mMetricsManagers.find(key);
		@@ -368,11 +380,17 @@ void StatsLogProcessor::WriteDataToDisk() {
		std::lock_guard<std::mutex> lock(mMetricsMutex);
		for (auto& pair : mMetricsManagers) {
		const ConfigKey& key = pair.first;
		vector<uint8_t> data;
		onDumpReportLocked(key, getElapsedRealtimeNs(), &data);
		ProtoOutputStream proto;
		onConfigMetricsReportLocked(key, getElapsedRealtimeNs(), &proto);
		string file_name = StringPrintf("%s/%ld_%d_%lld", STATS_DATA_DIR,
		(long)getWallClockSec(), key.GetUid(), (long long)key.GetId());
		StorageManager::writeFile(file_name.c_str(), &data[0], data.size());
		android::base::unique_fd fd(open(file_name.c_str(),
		O_WRONLY \| O_CREAT \| O_CLOEXEC, S_IRUSR \| S_IWUSR));
		if (fd == -1) {
		VLOG("Attempt to write %s but failed", file_name.c_str());
		return;
		}
		proto.flush(fd.get());
		}
		}

cmds/statsd/src/StatsLogProcessor.h

+2 −2

Original line number	Diff line number	Diff line
		@@ -86,8 +86,8 @@ private:

		sp<AlarmMonitor> mPeriodicAlarmMonitor;

		void onDumpReportLocked(const ConfigKey& key, const uint64_t dumpTimeNs,
		vector<uint8_t>* outData);
		void onConfigMetricsReportLocked(const ConfigKey& key, const uint64_t dumpTimeStampNs,
		util::ProtoOutputStream* proto);

		/* Check if we should send a broadcast if approaching memory limits and if we're over, we
		* actually delete the data. */

cmds/statsd/src/packages/UidMap.cpp

+32 −43

Original line number	Diff line number	Diff line
		@@ -331,25 +331,25 @@ size_t UidMap::getBytesUsed() const {
		return mBytesUsed;
		}

		void UidMap::getOutput(const ConfigKey& key, vector<uint8_t>* outData) {
		getOutput(getElapsedRealtimeNs(), key, outData);
		void UidMap::appendUidMap(const ConfigKey& key, ProtoOutputStream* proto) {
		appendUidMap(getElapsedRealtimeNs(), key, proto);
		}

		void UidMap::getOutput(const int64_t& timestamp, const ConfigKey& key, vector<uint8_t>* outData) {
		void UidMap::appendUidMap(const int64_t& timestamp, const ConfigKey& key,
		ProtoOutputStream* proto) {
		lock_guard<mutex> lock(mMutex); // Lock for updates

		ProtoOutputStream proto;
		for (const ChangeRecord& record : mChanges) {
		if (record.timestampNs > mLastUpdatePerConfigKey[key]) {
		uint64_t changesToken =
		proto.start(FIELD_TYPE_MESSAGE \| FIELD_COUNT_REPEATED \| FIELD_ID_CHANGES);
		proto.write(FIELD_TYPE_BOOL \| FIELD_ID_CHANGE_DELETION, (bool)record.deletion);
		proto.write(FIELD_TYPE_INT64 \| FIELD_ID_CHANGE_TIMESTAMP,
		proto->start(FIELD_TYPE_MESSAGE \| FIELD_COUNT_REPEATED \| FIELD_ID_CHANGES);
		proto->write(FIELD_TYPE_BOOL \| FIELD_ID_CHANGE_DELETION, (bool)record.deletion);
		proto->write(FIELD_TYPE_INT64 \| FIELD_ID_CHANGE_TIMESTAMP,
		(long long)record.timestampNs);
		proto.write(FIELD_TYPE_STRING \| FIELD_ID_CHANGE_PACKAGE, record.package);
		proto.write(FIELD_TYPE_INT32 \| FIELD_ID_CHANGE_UID, (int)record.uid);
		proto.write(FIELD_TYPE_INT32 \| FIELD_ID_CHANGE_VERSION, (int)record.version);
		proto.end(changesToken);
		proto->write(FIELD_TYPE_STRING \| FIELD_ID_CHANGE_PACKAGE, record.package);
		proto->write(FIELD_TYPE_INT32 \| FIELD_ID_CHANGE_UID, (int)record.uid);
		proto->write(FIELD_TYPE_INT32 \| FIELD_ID_CHANGE_VERSION, (int)record.version);
		proto->end(changesToken);
		}
		}

		@@ -360,13 +360,13 @@ void UidMap::getOutput(const int64_t& timestamp, const ConfigKey& key, vector<ui
		if ((count == mSnapshots.size() - 1 && !atLeastOneSnapshot) \|\|
		record.timestampNs > mLastUpdatePerConfigKey[key]) {
		uint64_t snapshotsToken =
		proto.start(FIELD_TYPE_MESSAGE \| FIELD_COUNT_REPEATED \| FIELD_ID_SNAPSHOTS);
		proto->start(FIELD_TYPE_MESSAGE \| FIELD_COUNT_REPEATED \| FIELD_ID_SNAPSHOTS);
		atLeastOneSnapshot = true;
		count++;
		proto.write(FIELD_TYPE_INT64 \| FIELD_ID_SNAPSHOT_TIMESTAMP,
		proto->write(FIELD_TYPE_INT64 \| FIELD_ID_SNAPSHOT_TIMESTAMP,
		(long long)record.timestampNs);
		proto.write(FIELD_TYPE_MESSAGE \| FIELD_ID_SNAPSHOT_PACKAGE_INFO, record.bytes.data());
		proto.end(snapshotsToken);
		proto->write(FIELD_TYPE_MESSAGE \| FIELD_ID_SNAPSHOT_PACKAGE_INFO, record.bytes.data());
		proto->end(snapshotsToken);
		}
		}

		@@ -398,47 +398,36 @@ void UidMap::getOutput(const int64_t& timestamp, const ConfigKey& key, vector<ui
		// Produce another snapshot. This results in extra data being uploaded but
		// helps ensure we can re-construct the UID->app name, versionCode mapping
		// in server.
		ProtoOutputStream proto;
		uint64_t token = proto.start(FIELD_TYPE_MESSAGE \| FIELD_COUNT_REPEATED \|
		ProtoOutputStream snapshotProto;
		uint64_t token = snapshotProto.start(FIELD_TYPE_MESSAGE \| FIELD_COUNT_REPEATED \|
		FIELD_ID_SNAPSHOT_PACKAGE_INFO);
		for (const auto& it : mMap) {
		proto.write(FIELD_TYPE_STRING \| FIELD_ID_SNAPSHOT_PACKAGE_NAME,
		snapshotProto.write(FIELD_TYPE_STRING \| FIELD_ID_SNAPSHOT_PACKAGE_NAME,
		it.second.packageName);
		proto.write(FIELD_TYPE_INT32 \| FIELD_ID_SNAPSHOT_PACKAGE_VERSION,
		snapshotProto.write(FIELD_TYPE_INT32 \| FIELD_ID_SNAPSHOT_PACKAGE_VERSION,
		(int)it.second.versionCode);
		proto.write(FIELD_TYPE_INT32 \| FIELD_ID_SNAPSHOT_PACKAGE_UID, (int)it.first);
		snapshotProto.write(FIELD_TYPE_INT32 \| FIELD_ID_SNAPSHOT_PACKAGE_UID,
		(int)it.first);
		}
		proto.end(token);
		snapshotProto.end(token);

		// Copy ProtoOutputStream output to
		auto iter = proto.data();
		vector<char> outData(proto.size());
		auto iter = snapshotProto.data();
		vector<char> snapshotData(snapshotProto.size());
		size_t pos = 0;
		while (iter.readBuffer() != NULL) {
		size_t toRead = iter.currentToRead();
		std::memcpy(&(outData[pos]), iter.readBuffer(), toRead);
		std::memcpy(&(snapshotData[pos]), iter.readBuffer(), toRead);
		pos += toRead;
		iter.rp()->move(toRead);
		}
		mSnapshots.emplace_back(timestamp, outData);
		mBytesUsed += kBytesTimestampField + outData.size();
		mSnapshots.emplace_back(timestamp, snapshotData);
		mBytesUsed += kBytesTimestampField + snapshotData.size();
		}
		}
		StatsdStats::getInstance().setCurrentUidMapMemory(mBytesUsed);
		StatsdStats::getInstance().setUidMapChanges(mChanges.size());
		StatsdStats::getInstance().setUidMapSnapshots(mSnapshots.size());
		if (outData != nullptr) {
		outData->clear();
		outData->resize(proto.size());
		size_t pos = 0;
		auto iter = proto.data();
		while (iter.readBuffer() != NULL) {
		size_t toRead = iter.currentToRead();
		std::memcpy(&((*outData)[pos]), iter.readBuffer(), toRead);
		pos += toRead;
		iter.rp()->move(toRead);
		}
		}
		}

		void UidMap::printUidMap(FILE* out) const {

cmds/statsd/src/packages/UidMap.h

+11 −6

Original line number	Diff line number	Diff line
		@@ -19,6 +19,7 @@
		#include "config/ConfigKey.h"
		#include "config/ConfigListener.h"
		#include "packages/PackageInfoListener.h"
		#include "stats_util.h"

		#include <binder/IResultReceiver.h>
		#include <binder/IShellCallback.h>
		@@ -32,8 +33,11 @@
		#include <string>
		#include <unordered_map>

		using namespace android;
		using namespace std;

		using android::util::ProtoOutputStream;

		namespace android {
		namespace os {
		namespace statsd {
		@@ -45,8 +49,8 @@ struct AppData {
		AppData(const string& a, const int64_t v) : packageName(a), versionCode(v){};
		};

		// When calling getOutput, we retrieve all the ChangeRecords since the last
		// timestamp we called getOutput for this configuration key.
		// When calling appendUidMap, we retrieve all the ChangeRecords since the last
		// timestamp we called appendUidMap for this configuration key.
		struct ChangeRecord {
		const bool deletion;
		const int64_t timestampNs;
		@@ -70,8 +74,8 @@ const unsigned int kBytesChangeRecord = sizeof(struct ChangeRecord);
		// less because of varint encoding).
		const unsigned int kBytesTimestampField = 10;

		// When calling getOutput, we retrieve all the snapshots since the last
		// timestamp we called getOutput for this configuration key.
		// When calling appendUidMap, we retrieve all the snapshots since the last
		// timestamp we called appendUidMap for this configuration key.
		struct SnapshotRecord {
		const int64_t timestampNs;

		@@ -135,7 +139,7 @@ public:
		// Gets all snapshots and changes that have occurred since the last output.
		// If every config key has received a change or snapshot record, then this
		// record is deleted.
		void getOutput(const ConfigKey& key, vector<uint8_t>* outData);
		void appendUidMap(const ConfigKey& key, util::ProtoOutputStream* proto);

		// Forces the output to be cleared. We still generate a snapshot based on the current state.
		// This results in extra data uploaded but helps us reconstruct the uid mapping on the server
		@@ -158,7 +162,8 @@ private:
		const int64_t& versionCode);
		void removeApp(const int64_t& timestamp, const String16& packageName, const int32_t& uid);

		void getOutput(const int64_t& timestamp, const ConfigKey& key, vector<uint8_t>* outData);
		void appendUidMap(const int64_t& timestamp, const ConfigKey& key,
		util::ProtoOutputStream* proto);

		void getListenerListCopyLocked(std::vector<wp<PackageInfoListener>>* output);

cmds/statsd/src/storage/StorageManager.cpp

+29 −19

Original line number	Diff line number	Diff line
		@@ -160,32 +160,39 @@ void StorageManager::sendBroadcast(const char* path,
		}
		}

		void StorageManager::appendConfigMetricsReport(ProtoOutputStream& proto) {
		void StorageManager::appendConfigMetricsReport(const ConfigKey& key, ProtoOutputStream* proto) {
		unique_ptr<DIR, decltype(&closedir)> dir(opendir(STATS_DATA_DIR), closedir);
		if (dir == NULL) {
		VLOG("Path %s does not exist", STATS_DATA_DIR);
		return;
		}

		const char* suffix =
		StringPrintf("%d_%lld", key.GetUid(), (long long)key.GetId()).c_str();

		dirent* de;
		while ((de = readdir(dir.get()))) {
		char* name = de->d_name;
		if (name[0] == '.') continue;
		VLOG("file %s", name);

		size_t nameLen = strlen(name);
		size_t suffixLen = strlen(suffix);
		if (suffixLen <= nameLen &&
		strncmp(name + nameLen - suffixLen, suffix, suffixLen) == 0) {
		int64_t result[3];
		parseFileName(name, result);
		if (result[0] == -1) continue;
		int64_t timestamp = result[0];
		int64_t uid = result[1];
		int64_t configID = result[2];

		string file_name = getFilePath(STATS_DATA_DIR, timestamp, uid, configID);
		int fd = open(file_name.c_str(), O_RDONLY \| O_CLOEXEC);
		if (fd != -1) {
		string content;
		if (android::base::ReadFdToString(fd, &content)) {
		proto.write(FIELD_TYPE_MESSAGE \| FIELD_COUNT_REPEATED \| FIELD_ID_REPORTS,
		content.c_str());
		proto->write(FIELD_TYPE_MESSAGE \| FIELD_COUNT_REPEATED \| FIELD_ID_REPORTS,
		content.c_str(), content.size());
		}
		close(fd);
		}
		@@ -194,6 +201,7 @@ void StorageManager::appendConfigMetricsReport(ProtoOutputStream& proto) {
		remove(file_name.c_str());
		}
		}
		}

		bool StorageManager::readFileToString(const char* file, string* content) {
		int fd = open(file, O_RDONLY \| O_CLOEXEC);
		@@ -275,9 +283,11 @@ bool StorageManager::hasIdenticalConfig(const ConfigKey& key,
		if (android::base::ReadFdToString(fd, &content)) {
		vector<uint8_t> vec(content.begin(), content.end());
		if (vec == config) {
		close(fd);
		return true;
		}
		}
		close(fd);
		}
		}
		}