Merge "Fix recovery of stats data from previous input while using ProtoOutputStream" into pi-dev (e51af374) · Commits · e / os / android_frameworks_base

cmds/statsd/src/StatsLogProcessor.cpp

+50 −32

Original line number	Original line	Diff line number	Diff line
	@@ -239,14 +239,13 @@ void StatsLogProcessor::dumpStates(FILE* out, bool verbose) {
	}		}
	}		}

			/*
			* onDumpReport dumps serialized ConfigMetricsReportList into outData.
			*/
	void StatsLogProcessor::onDumpReport(const ConfigKey& key, const uint64_t dumpTimeStampNs,		void StatsLogProcessor::onDumpReport(const ConfigKey& key, const uint64_t dumpTimeStampNs,
	vector<uint8_t>* outData) {		vector<uint8_t>* outData) {
	std::lock_guard<std::mutex> lock(mMetricsMutex);		std::lock_guard<std::mutex> lock(mMetricsMutex);
	onDumpReportLocked(key, dumpTimeStampNs, outData);
	}

	void StatsLogProcessor::onDumpReportLocked(const ConfigKey& key, const uint64_t dumpTimeStampNs,
	vector<uint8_t>* outData) {
	auto it = mMetricsManagers.find(key);		auto it = mMetricsManagers.find(key);
	if (it == mMetricsManagers.end()) {		if (it == mMetricsManagers.end()) {
	ALOGW("Config source %s does not exist", key.ToString().c_str());		ALOGW("Config source %s does not exist", key.ToString().c_str());
	@@ -269,35 +268,14 @@ void StatsLogProcessor::onDumpReportLocked(const ConfigKey& key, const uint64_t
	// Start of ConfigMetricsReport (reports).		// Start of ConfigMetricsReport (reports).
	uint64_t reportsToken =		uint64_t reportsToken =
	proto.start(FIELD_TYPE_MESSAGE \| FIELD_COUNT_REPEATED \| FIELD_ID_REPORTS);		proto.start(FIELD_TYPE_MESSAGE \| FIELD_COUNT_REPEATED \| FIELD_ID_REPORTS);
			onConfigMetricsReportLocked(key, dumpTimeStampNs, &proto);
	int64_t lastReportTimeNs = it->second->getLastReportTimeNs();
	int64_t lastReportWallClockNs = it->second->getLastReportWallClockNs();

	// First, fill in ConfigMetricsReport using current data on memory, which
	// starts from filling in StatsLogReport's.
	it->second->onDumpReport(dumpTimeStampNs, &proto);

	// Fill in UidMap.
	vector<uint8_t> uidMap;
	mUidMap->getOutput(key, &uidMap);
	proto.write(FIELD_TYPE_MESSAGE \| FIELD_ID_UID_MAP, uidMap.data());

	// Fill in the timestamps.
	proto.write(FIELD_TYPE_INT64 \| FIELD_ID_LAST_REPORT_ELAPSED_NANOS,
	(long long)lastReportTimeNs);
	proto.write(FIELD_TYPE_INT64 \| FIELD_ID_CURRENT_REPORT_ELAPSED_NANOS,
	(long long)dumpTimeStampNs);
	proto.write(FIELD_TYPE_INT64 \| FIELD_ID_LAST_REPORT_WALL_CLOCK_NANOS,
	(long long)lastReportWallClockNs);
	proto.write(FIELD_TYPE_INT64 \| FIELD_ID_CURRENT_REPORT_WALL_CLOCK_NANOS,
	(long long)getWallClockNs());

	// End of ConfigMetricsReport (reports).
	proto.end(reportsToken);		proto.end(reportsToken);
			// End of ConfigMetricsReport (reports).


	// Then, check stats-data directory to see there's any file containing		// Then, check stats-data directory to see there's any file containing
	// ConfigMetricsReport from previous shutdowns to concatenate to reports.		// ConfigMetricsReport from previous shutdowns to concatenate to reports.
	StorageManager::appendConfigMetricsReport(proto);		StorageManager::appendConfigMetricsReport(key, &proto);

	if (outData != nullptr) {		if (outData != nullptr) {
	outData->clear();		outData->clear();
	@@ -315,6 +293,40 @@ void StatsLogProcessor::onDumpReportLocked(const ConfigKey& key, const uint64_t
	StatsdStats::getInstance().noteMetricsReportSent(key);		StatsdStats::getInstance().noteMetricsReportSent(key);
	}		}

			/*
			* onConfigMetricsReportLocked dumps serialized ConfigMetricsReport into outData.
			*/
			void StatsLogProcessor::onConfigMetricsReportLocked(const ConfigKey& key,
			const uint64_t dumpTimeStampNs,
			ProtoOutputStream* proto) {
			// We already checked whether key exists in mMetricsManagers in
			// WriteDataToDisk.
			auto it = mMetricsManagers.find(key);
			int64_t lastReportTimeNs = it->second->getLastReportTimeNs();
			int64_t lastReportWallClockNs = it->second->getLastReportWallClockNs();

			// First, fill in ConfigMetricsReport using current data on memory, which
			// starts from filling in StatsLogReport's.
			it->second->onDumpReport(dumpTimeStampNs, proto);

			// Fill in UidMap.
			uint64_t uidMapToken = proto->start(FIELD_TYPE_MESSAGE \| FIELD_ID_UID_MAP);
			mUidMap->appendUidMap(key, proto);
			proto->end(uidMapToken);

			// Fill in the timestamps.
			proto->write(FIELD_TYPE_INT64 \| FIELD_ID_LAST_REPORT_ELAPSED_NANOS,
			(long long)lastReportTimeNs);
			proto->write(FIELD_TYPE_INT64 \| FIELD_ID_CURRENT_REPORT_ELAPSED_NANOS,
			(long long)dumpTimeStampNs);
			proto->write(FIELD_TYPE_INT64 \| FIELD_ID_LAST_REPORT_WALL_CLOCK_NANOS,
			(long long)lastReportWallClockNs);
			proto->write(FIELD_TYPE_INT64 \| FIELD_ID_CURRENT_REPORT_WALL_CLOCK_NANOS,
			(long long)getWallClockNs());


			}

	void StatsLogProcessor::OnConfigRemoved(const ConfigKey& key) {		void StatsLogProcessor::OnConfigRemoved(const ConfigKey& key) {
	std::lock_guard<std::mutex> lock(mMetricsMutex);		std::lock_guard<std::mutex> lock(mMetricsMutex);
	auto it = mMetricsManagers.find(key);		auto it = mMetricsManagers.find(key);
	@@ -368,11 +380,17 @@ void StatsLogProcessor::WriteDataToDisk() {
	std::lock_guard<std::mutex> lock(mMetricsMutex);		std::lock_guard<std::mutex> lock(mMetricsMutex);
	for (auto& pair : mMetricsManagers) {		for (auto& pair : mMetricsManagers) {
	const ConfigKey& key = pair.first;		const ConfigKey& key = pair.first;
	vector<uint8_t> data;		ProtoOutputStream proto;
	onDumpReportLocked(key, getElapsedRealtimeNs(), &data);		onConfigMetricsReportLocked(key, getElapsedRealtimeNs(), &proto);
	string file_name = StringPrintf("%s/%ld_%d_%lld", STATS_DATA_DIR,		string file_name = StringPrintf("%s/%ld_%d_%lld", STATS_DATA_DIR,
	(long)getWallClockSec(), key.GetUid(), (long long)key.GetId());		(long)getWallClockSec(), key.GetUid(), (long long)key.GetId());
	StorageManager::writeFile(file_name.c_str(), &data[0], data.size());		android::base::unique_fd fd(open(file_name.c_str(),
			O_WRONLY \| O_CREAT \| O_CLOEXEC, S_IRUSR \| S_IWUSR));
			if (fd == -1) {
			VLOG("Attempt to write %s but failed", file_name.c_str());
			return;
			}
			proto.flush(fd.get());
	}		}
	}		}

cmds/statsd/src/StatsLogProcessor.h

+2 −2

Original line number	Original line	Diff line number	Diff line
	@@ -86,8 +86,8 @@ private:

	sp<AlarmMonitor> mPeriodicAlarmMonitor;		sp<AlarmMonitor> mPeriodicAlarmMonitor;

	void onDumpReportLocked(const ConfigKey& key, const uint64_t dumpTimeNs,		void onConfigMetricsReportLocked(const ConfigKey& key, const uint64_t dumpTimeStampNs,
	vector<uint8_t>* outData);		util::ProtoOutputStream* proto);

	/* Check if we should send a broadcast if approaching memory limits and if we're over, we		/* Check if we should send a broadcast if approaching memory limits and if we're over, we
	* actually delete the data. */		* actually delete the data. */

cmds/statsd/src/packages/UidMap.cpp

+32 −43

Original line number	Original line	Diff line number	Diff line
	@@ -331,25 +331,25 @@ size_t UidMap::getBytesUsed() const {
	return mBytesUsed;		return mBytesUsed;
	}		}

	void UidMap::getOutput(const ConfigKey& key, vector<uint8_t>* outData) {		void UidMap::appendUidMap(const ConfigKey& key, ProtoOutputStream* proto) {
	getOutput(getElapsedRealtimeNs(), key, outData);		appendUidMap(getElapsedRealtimeNs(), key, proto);
	}		}

	void UidMap::getOutput(const int64_t& timestamp, const ConfigKey& key, vector<uint8_t>* outData) {		void UidMap::appendUidMap(const int64_t& timestamp, const ConfigKey& key,
			ProtoOutputStream* proto) {
	lock_guard<mutex> lock(mMutex); // Lock for updates		lock_guard<mutex> lock(mMutex); // Lock for updates

	ProtoOutputStream proto;
	for (const ChangeRecord& record : mChanges) {		for (const ChangeRecord& record : mChanges) {
	if (record.timestampNs > mLastUpdatePerConfigKey[key]) {		if (record.timestampNs > mLastUpdatePerConfigKey[key]) {
	uint64_t changesToken =		uint64_t changesToken =
	proto.start(FIELD_TYPE_MESSAGE \| FIELD_COUNT_REPEATED \| FIELD_ID_CHANGES);		proto->start(FIELD_TYPE_MESSAGE \| FIELD_COUNT_REPEATED \| FIELD_ID_CHANGES);
	proto.write(FIELD_TYPE_BOOL \| FIELD_ID_CHANGE_DELETION, (bool)record.deletion);		proto->write(FIELD_TYPE_BOOL \| FIELD_ID_CHANGE_DELETION, (bool)record.deletion);
	proto.write(FIELD_TYPE_INT64 \| FIELD_ID_CHANGE_TIMESTAMP,		proto->write(FIELD_TYPE_INT64 \| FIELD_ID_CHANGE_TIMESTAMP,
	(long long)record.timestampNs);		(long long)record.timestampNs);
	proto.write(FIELD_TYPE_STRING \| FIELD_ID_CHANGE_PACKAGE, record.package);		proto->write(FIELD_TYPE_STRING \| FIELD_ID_CHANGE_PACKAGE, record.package);
	proto.write(FIELD_TYPE_INT32 \| FIELD_ID_CHANGE_UID, (int)record.uid);		proto->write(FIELD_TYPE_INT32 \| FIELD_ID_CHANGE_UID, (int)record.uid);
	proto.write(FIELD_TYPE_INT32 \| FIELD_ID_CHANGE_VERSION, (int)record.version);		proto->write(FIELD_TYPE_INT32 \| FIELD_ID_CHANGE_VERSION, (int)record.version);
	proto.end(changesToken);		proto->end(changesToken);
	}		}
	}		}

	@@ -360,13 +360,13 @@ void UidMap::getOutput(const int64_t& timestamp, const ConfigKey& key, vector<ui
	if ((count == mSnapshots.size() - 1 && !atLeastOneSnapshot) \|\|		if ((count == mSnapshots.size() - 1 && !atLeastOneSnapshot) \|\|
	record.timestampNs > mLastUpdatePerConfigKey[key]) {		record.timestampNs > mLastUpdatePerConfigKey[key]) {
	uint64_t snapshotsToken =		uint64_t snapshotsToken =
	proto.start(FIELD_TYPE_MESSAGE \| FIELD_COUNT_REPEATED \| FIELD_ID_SNAPSHOTS);		proto->start(FIELD_TYPE_MESSAGE \| FIELD_COUNT_REPEATED \| FIELD_ID_SNAPSHOTS);
	atLeastOneSnapshot = true;		atLeastOneSnapshot = true;
	count++;		count++;
	proto.write(FIELD_TYPE_INT64 \| FIELD_ID_SNAPSHOT_TIMESTAMP,		proto->write(FIELD_TYPE_INT64 \| FIELD_ID_SNAPSHOT_TIMESTAMP,
	(long long)record.timestampNs);		(long long)record.timestampNs);
	proto.write(FIELD_TYPE_MESSAGE \| FIELD_ID_SNAPSHOT_PACKAGE_INFO, record.bytes.data());		proto->write(FIELD_TYPE_MESSAGE \| FIELD_ID_SNAPSHOT_PACKAGE_INFO, record.bytes.data());
	proto.end(snapshotsToken);		proto->end(snapshotsToken);
	}		}
	}		}

	@@ -398,47 +398,36 @@ void UidMap::getOutput(const int64_t& timestamp, const ConfigKey& key, vector<ui
	// Produce another snapshot. This results in extra data being uploaded but		// Produce another snapshot. This results in extra data being uploaded but
	// helps ensure we can re-construct the UID->app name, versionCode mapping		// helps ensure we can re-construct the UID->app name, versionCode mapping
	// in server.		// in server.
	ProtoOutputStream proto;		ProtoOutputStream snapshotProto;
	uint64_t token = proto.start(FIELD_TYPE_MESSAGE \| FIELD_COUNT_REPEATED \|		uint64_t token = snapshotProto.start(FIELD_TYPE_MESSAGE \| FIELD_COUNT_REPEATED \|
	FIELD_ID_SNAPSHOT_PACKAGE_INFO);		FIELD_ID_SNAPSHOT_PACKAGE_INFO);
	for (const auto& it : mMap) {		for (const auto& it : mMap) {
	proto.write(FIELD_TYPE_STRING \| FIELD_ID_SNAPSHOT_PACKAGE_NAME,		snapshotProto.write(FIELD_TYPE_STRING \| FIELD_ID_SNAPSHOT_PACKAGE_NAME,
	it.second.packageName);		it.second.packageName);
	proto.write(FIELD_TYPE_INT32 \| FIELD_ID_SNAPSHOT_PACKAGE_VERSION,		snapshotProto.write(FIELD_TYPE_INT32 \| FIELD_ID_SNAPSHOT_PACKAGE_VERSION,
	(int)it.second.versionCode);		(int)it.second.versionCode);
	proto.write(FIELD_TYPE_INT32 \| FIELD_ID_SNAPSHOT_PACKAGE_UID, (int)it.first);		snapshotProto.write(FIELD_TYPE_INT32 \| FIELD_ID_SNAPSHOT_PACKAGE_UID,
			(int)it.first);
	}		}
	proto.end(token);		snapshotProto.end(token);

	// Copy ProtoOutputStream output to		// Copy ProtoOutputStream output to
	auto iter = proto.data();		auto iter = snapshotProto.data();
	vector<char> outData(proto.size());		vector<char> snapshotData(snapshotProto.size());
	size_t pos = 0;		size_t pos = 0;
	while (iter.readBuffer() != NULL) {		while (iter.readBuffer() != NULL) {
	size_t toRead = iter.currentToRead();		size_t toRead = iter.currentToRead();
	std::memcpy(&(outData[pos]), iter.readBuffer(), toRead);		std::memcpy(&(snapshotData[pos]), iter.readBuffer(), toRead);
	pos += toRead;		pos += toRead;
	iter.rp()->move(toRead);		iter.rp()->move(toRead);
	}		}
	mSnapshots.emplace_back(timestamp, outData);		mSnapshots.emplace_back(timestamp, snapshotData);
	mBytesUsed += kBytesTimestampField + outData.size();		mBytesUsed += kBytesTimestampField + snapshotData.size();
	}		}
	}		}
	StatsdStats::getInstance().setCurrentUidMapMemory(mBytesUsed);		StatsdStats::getInstance().setCurrentUidMapMemory(mBytesUsed);
	StatsdStats::getInstance().setUidMapChanges(mChanges.size());		StatsdStats::getInstance().setUidMapChanges(mChanges.size());
	StatsdStats::getInstance().setUidMapSnapshots(mSnapshots.size());		StatsdStats::getInstance().setUidMapSnapshots(mSnapshots.size());
	if (outData != nullptr) {
	outData->clear();
	outData->resize(proto.size());
	size_t pos = 0;
	auto iter = proto.data();
	while (iter.readBuffer() != NULL) {
	size_t toRead = iter.currentToRead();
	std::memcpy(&((*outData)[pos]), iter.readBuffer(), toRead);
	pos += toRead;
	iter.rp()->move(toRead);
	}
	}
	}		}

	void UidMap::printUidMap(FILE* out) const {		void UidMap::printUidMap(FILE* out) const {

cmds/statsd/src/packages/UidMap.h

+11 −6

Original line number	Original line	Diff line number	Diff line
	@@ -19,6 +19,7 @@
	#include "config/ConfigKey.h"		#include "config/ConfigKey.h"
	#include "config/ConfigListener.h"		#include "config/ConfigListener.h"
	#include "packages/PackageInfoListener.h"		#include "packages/PackageInfoListener.h"
			#include "stats_util.h"

	#include <binder/IResultReceiver.h>		#include <binder/IResultReceiver.h>
	#include <binder/IShellCallback.h>		#include <binder/IShellCallback.h>
	@@ -32,8 +33,11 @@
	#include <string>		#include <string>
	#include <unordered_map>		#include <unordered_map>

			using namespace android;
	using namespace std;		using namespace std;

			using android::util::ProtoOutputStream;

	namespace android {		namespace android {
	namespace os {		namespace os {
	namespace statsd {		namespace statsd {
	@@ -45,8 +49,8 @@ struct AppData {
	AppData(const string& a, const int64_t v) : packageName(a), versionCode(v){};		AppData(const string& a, const int64_t v) : packageName(a), versionCode(v){};
	};		};

	// When calling getOutput, we retrieve all the ChangeRecords since the last		// When calling appendUidMap, we retrieve all the ChangeRecords since the last
	// timestamp we called getOutput for this configuration key.		// timestamp we called appendUidMap for this configuration key.
	struct ChangeRecord {		struct ChangeRecord {
	const bool deletion;		const bool deletion;
	const int64_t timestampNs;		const int64_t timestampNs;
	@@ -70,8 +74,8 @@ const unsigned int kBytesChangeRecord = sizeof(struct ChangeRecord);
	// less because of varint encoding).		// less because of varint encoding).
	const unsigned int kBytesTimestampField = 10;		const unsigned int kBytesTimestampField = 10;

	// When calling getOutput, we retrieve all the snapshots since the last		// When calling appendUidMap, we retrieve all the snapshots since the last
	// timestamp we called getOutput for this configuration key.		// timestamp we called appendUidMap for this configuration key.
	struct SnapshotRecord {		struct SnapshotRecord {
	const int64_t timestampNs;		const int64_t timestampNs;

	@@ -135,7 +139,7 @@ public:
	// Gets all snapshots and changes that have occurred since the last output.		// Gets all snapshots and changes that have occurred since the last output.
	// If every config key has received a change or snapshot record, then this		// If every config key has received a change or snapshot record, then this
	// record is deleted.		// record is deleted.
	void getOutput(const ConfigKey& key, vector<uint8_t>* outData);		void appendUidMap(const ConfigKey& key, util::ProtoOutputStream* proto);

	// Forces the output to be cleared. We still generate a snapshot based on the current state.		// Forces the output to be cleared. We still generate a snapshot based on the current state.
	// This results in extra data uploaded but helps us reconstruct the uid mapping on the server		// This results in extra data uploaded but helps us reconstruct the uid mapping on the server
	@@ -158,7 +162,8 @@ private:
	const int64_t& versionCode);		const int64_t& versionCode);
	void removeApp(const int64_t& timestamp, const String16& packageName, const int32_t& uid);		void removeApp(const int64_t& timestamp, const String16& packageName, const int32_t& uid);

	void getOutput(const int64_t& timestamp, const ConfigKey& key, vector<uint8_t>* outData);		void appendUidMap(const int64_t& timestamp, const ConfigKey& key,
			util::ProtoOutputStream* proto);

	void getListenerListCopyLocked(std::vector<wp<PackageInfoListener>>* output);		void getListenerListCopyLocked(std::vector<wp<PackageInfoListener>>* output);

cmds/statsd/src/storage/StorageManager.cpp

+29 −19

Original line number	Original line	Diff line number	Diff line
	@@ -160,32 +160,39 @@ void StorageManager::sendBroadcast(const char* path,
	}		}
	}		}

	void StorageManager::appendConfigMetricsReport(ProtoOutputStream& proto) {		void StorageManager::appendConfigMetricsReport(const ConfigKey& key, ProtoOutputStream* proto) {
	unique_ptr<DIR, decltype(&closedir)> dir(opendir(STATS_DATA_DIR), closedir);		unique_ptr<DIR, decltype(&closedir)> dir(opendir(STATS_DATA_DIR), closedir);
	if (dir == NULL) {		if (dir == NULL) {
	VLOG("Path %s does not exist", STATS_DATA_DIR);		VLOG("Path %s does not exist", STATS_DATA_DIR);
	return;		return;
	}		}

			const char* suffix =
			StringPrintf("%d_%lld", key.GetUid(), (long long)key.GetId()).c_str();

	dirent* de;		dirent* de;
	while ((de = readdir(dir.get()))) {		while ((de = readdir(dir.get()))) {
	char* name = de->d_name;		char* name = de->d_name;
	if (name[0] == '.') continue;		if (name[0] == '.') continue;
	VLOG("file %s", name);

			size_t nameLen = strlen(name);
			size_t suffixLen = strlen(suffix);
			if (suffixLen <= nameLen &&
			strncmp(name + nameLen - suffixLen, suffix, suffixLen) == 0) {
	int64_t result[3];		int64_t result[3];
	parseFileName(name, result);		parseFileName(name, result);
	if (result[0] == -1) continue;		if (result[0] == -1) continue;
	int64_t timestamp = result[0];		int64_t timestamp = result[0];
	int64_t uid = result[1];		int64_t uid = result[1];
	int64_t configID = result[2];		int64_t configID = result[2];

	string file_name = getFilePath(STATS_DATA_DIR, timestamp, uid, configID);		string file_name = getFilePath(STATS_DATA_DIR, timestamp, uid, configID);
	int fd = open(file_name.c_str(), O_RDONLY \| O_CLOEXEC);		int fd = open(file_name.c_str(), O_RDONLY \| O_CLOEXEC);
	if (fd != -1) {		if (fd != -1) {
	string content;		string content;
	if (android::base::ReadFdToString(fd, &content)) {		if (android::base::ReadFdToString(fd, &content)) {
	proto.write(FIELD_TYPE_MESSAGE \| FIELD_COUNT_REPEATED \| FIELD_ID_REPORTS,		proto->write(FIELD_TYPE_MESSAGE \| FIELD_COUNT_REPEATED \| FIELD_ID_REPORTS,
	content.c_str());		content.c_str(), content.size());
	}		}
	close(fd);		close(fd);
	}		}
	@@ -194,6 +201,7 @@ void StorageManager::appendConfigMetricsReport(ProtoOutputStream& proto) {
	remove(file_name.c_str());		remove(file_name.c_str());
	}		}
	}		}
			}

	bool StorageManager::readFileToString(const char* file, string* content) {		bool StorageManager::readFileToString(const char* file, string* content) {
	int fd = open(file, O_RDONLY \| O_CLOEXEC);		int fd = open(file, O_RDONLY \| O_CLOEXEC);
	@@ -275,9 +283,11 @@ bool StorageManager::hasIdenticalConfig(const ConfigKey& key,
	if (android::base::ReadFdToString(fd, &content)) {		if (android::base::ReadFdToString(fd, &content)) {
	vector<uint8_t> vec(content.begin(), content.end());		vector<uint8_t> vec(content.begin(), content.end());
	if (vec == config) {		if (vec == config) {
			close(fd);
	return true;		return true;
	}		}
	}		}
			close(fd);
	}		}
	}		}
	}		}