Merge "Flush the partial bucket when startd shuts down or config updated." into pi-dev (9d4c3977) · Commits · e / os / android_frameworks_base

cmds/statsd/src/StatsLogProcessor.cpp

+50 −36

Original line number	Original line	Diff line number	Diff line
	@@ -212,7 +212,10 @@ void StatsLogProcessor::OnConfigUpdatedLocked(
	sp<MetricsManager> newMetricsManager =		sp<MetricsManager> newMetricsManager =
	new MetricsManager(key, config, mTimeBaseSec, (timestampNs - 1) / NS_PER_SEC + 1, mUidMap,		new MetricsManager(key, config, mTimeBaseSec, (timestampNs - 1) / NS_PER_SEC + 1, mUidMap,
	mAnomalyAlarmMonitor, mPeriodicAlarmMonitor);		mAnomalyAlarmMonitor, mPeriodicAlarmMonitor);
			auto it = mMetricsManagers.find(key);
			if (it != mMetricsManagers.end()) {
			WriteDataToDiskLocked(it->first);
			}
	if (newMetricsManager->isConfigValid()) {		if (newMetricsManager->isConfigValid()) {
	mUidMap->OnConfigUpdated(key);		mUidMap->OnConfigUpdated(key);
	if (newMetricsManager->shouldAddUidMapListener()) {		if (newMetricsManager->shouldAddUidMapListener()) {
	@@ -251,19 +254,10 @@ void StatsLogProcessor::dumpStates(FILE* out, bool verbose) {
	* onDumpReport dumps serialized ConfigMetricsReportList into outData.		* onDumpReport dumps serialized ConfigMetricsReportList into outData.
	*/		*/
	void StatsLogProcessor::onDumpReport(const ConfigKey& key, const int64_t dumpTimeStampNs,		void StatsLogProcessor::onDumpReport(const ConfigKey& key, const int64_t dumpTimeStampNs,
			const bool include_current_partial_bucket,
	vector<uint8_t>* outData) {		vector<uint8_t>* outData) {
	std::lock_guard<std::mutex> lock(mMetricsMutex);		std::lock_guard<std::mutex> lock(mMetricsMutex);

	auto it = mMetricsManagers.find(key);
	if (it == mMetricsManagers.end()) {
	ALOGW("Config source %s does not exist", key.ToString().c_str());
	return;
	}

	// This allows another broadcast to be sent within the rate-limit period if we get close to
	// filling the buffer again soon.
	mLastBroadcastTimes.erase(key);

	ProtoOutputStream proto;		ProtoOutputStream proto;

	// Start of ConfigKey.		// Start of ConfigKey.
	@@ -273,17 +267,25 @@ void StatsLogProcessor::onDumpReport(const ConfigKey& key, const int64_t dumpTim
	proto.end(configKeyToken);		proto.end(configKeyToken);
	// End of ConfigKey.		// End of ConfigKey.

			// Then, check stats-data directory to see there's any file containing
			// ConfigMetricsReport from previous shutdowns to concatenate to reports.
			StorageManager::appendConfigMetricsReport(key, &proto);

			auto it = mMetricsManagers.find(key);
			if (it != mMetricsManagers.end()) {
			// This allows another broadcast to be sent within the rate-limit period if we get close to
			// filling the buffer again soon.
			mLastBroadcastTimes.erase(key);

	// Start of ConfigMetricsReport (reports).		// Start of ConfigMetricsReport (reports).
	uint64_t reportsToken =		uint64_t reportsToken =
	proto.start(FIELD_TYPE_MESSAGE \| FIELD_COUNT_REPEATED \| FIELD_ID_REPORTS);		proto.start(FIELD_TYPE_MESSAGE \| FIELD_COUNT_REPEATED \| FIELD_ID_REPORTS);
	onConfigMetricsReportLocked(key, dumpTimeStampNs, &proto);		onConfigMetricsReportLocked(key, dumpTimeStampNs, include_current_partial_bucket, &proto);
	proto.end(reportsToken);		proto.end(reportsToken);
	// End of ConfigMetricsReport (reports).		// End of ConfigMetricsReport (reports).
			} else {
			ALOGW("Config source %s does not exist", key.ToString().c_str());
	// Then, check stats-data directory to see there's any file containing		}
	// ConfigMetricsReport from previous shutdowns to concatenate to reports.
	StorageManager::appendConfigMetricsReport(key, &proto);

	if (outData != nullptr) {		if (outData != nullptr) {
	outData->clear();		outData->clear();
	@@ -298,7 +300,7 @@ void StatsLogProcessor::onDumpReport(const ConfigKey& key, const int64_t dumpTim
	}		}
	}		}

	StatsdStats::getInstance().noteMetricsReportSent(key);		StatsdStats::getInstance().noteMetricsReportSent(key, proto.size());
	}		}

	/*		/*
	@@ -306,16 +308,20 @@ void StatsLogProcessor::onDumpReport(const ConfigKey& key, const int64_t dumpTim
	*/		*/
	void StatsLogProcessor::onConfigMetricsReportLocked(const ConfigKey& key,		void StatsLogProcessor::onConfigMetricsReportLocked(const ConfigKey& key,
	const int64_t dumpTimeStampNs,		const int64_t dumpTimeStampNs,
			const bool include_current_partial_bucket,
	ProtoOutputStream* proto) {		ProtoOutputStream* proto) {
	// We already checked whether key exists in mMetricsManagers in		// We already checked whether key exists in mMetricsManagers in
	// WriteDataToDisk.		// WriteDataToDisk.
	auto it = mMetricsManagers.find(key);		auto it = mMetricsManagers.find(key);
			if (it == mMetricsManagers.end()) {
			return;
			}
	int64_t lastReportTimeNs = it->second->getLastReportTimeNs();		int64_t lastReportTimeNs = it->second->getLastReportTimeNs();
	int64_t lastReportWallClockNs = it->second->getLastReportWallClockNs();		int64_t lastReportWallClockNs = it->second->getLastReportWallClockNs();

	// First, fill in ConfigMetricsReport using current data on memory, which		// First, fill in ConfigMetricsReport using current data on memory, which
	// starts from filling in StatsLogReport's.		// starts from filling in StatsLogReport's.
	it->second->onDumpReport(dumpTimeStampNs, proto);		it->second->onDumpReport(dumpTimeStampNs, include_current_partial_bucket, proto);

	// Fill in UidMap.		// Fill in UidMap.
	uint64_t uidMapToken = proto->start(FIELD_TYPE_MESSAGE \| FIELD_ID_UID_MAP);		uint64_t uidMapToken = proto->start(FIELD_TYPE_MESSAGE \| FIELD_ID_UID_MAP);
	@@ -331,7 +337,6 @@ void StatsLogProcessor::onConfigMetricsReportLocked(const ConfigKey& key,
	(long long)lastReportWallClockNs);		(long long)lastReportWallClockNs);
	proto->write(FIELD_TYPE_INT64 \| FIELD_ID_CURRENT_REPORT_WALL_CLOCK_NANOS,		proto->write(FIELD_TYPE_INT64 \| FIELD_ID_CURRENT_REPORT_WALL_CLOCK_NANOS,
	(long long)getWallClockNs());		(long long)getWallClockNs());

	}		}

	void StatsLogProcessor::resetIfConfigTtlExpiredLocked(const int64_t timestampNs) {		void StatsLogProcessor::resetIfConfigTtlExpiredLocked(const int64_t timestampNs) {
	@@ -361,6 +366,7 @@ void StatsLogProcessor::OnConfigRemoved(const ConfigKey& key) {
	std::lock_guard<std::mutex> lock(mMetricsMutex);		std::lock_guard<std::mutex> lock(mMetricsMutex);
	auto it = mMetricsManagers.find(key);		auto it = mMetricsManagers.find(key);
	if (it != mMetricsManagers.end()) {		if (it != mMetricsManagers.end()) {
			WriteDataToDiskLocked(key);
	mMetricsManagers.erase(it);		mMetricsManagers.erase(it);
	mUidMap->OnConfigRemoved(key);		mUidMap->OnConfigRemoved(key);
	}		}
	@@ -406,22 +412,30 @@ void StatsLogProcessor::flushIfNecessaryLocked(
	}		}
	}		}

	void StatsLogProcessor::WriteDataToDisk() {		void StatsLogProcessor::WriteDataToDiskLocked(const ConfigKey& key) {
	std::lock_guard<std::mutex> lock(mMetricsMutex);
	for (auto& pair : mMetricsManagers) {
	const ConfigKey& key = pair.first;
	ProtoOutputStream proto;		ProtoOutputStream proto;
	onConfigMetricsReportLocked(key, getElapsedRealtimeNs(), &proto);		onConfigMetricsReportLocked(key, getElapsedRealtimeNs(),
			true /* include_current_partial_bucket*/, &proto);
	string file_name = StringPrintf("%s/%ld_%d_%lld", STATS_DATA_DIR,		string file_name = StringPrintf("%s/%ld_%d_%lld", STATS_DATA_DIR,
	(long)getWallClockSec(), key.GetUid(), (long long)key.GetId());		(long)getWallClockSec(), key.GetUid(), (long long)key.GetId());
	android::base::unique_fd fd(open(file_name.c_str(),		android::base::unique_fd fd(open(file_name.c_str(),
	O_WRONLY \| O_CREAT \| O_CLOEXEC, S_IRUSR \| S_IWUSR));		O_WRONLY \| O_CREAT \| O_CLOEXEC, S_IRUSR \| S_IWUSR));
	if (fd == -1) {		if (fd == -1) {
	VLOG("Attempt to write %s but failed", file_name.c_str());		ALOGE("Attempt to write %s but failed", file_name.c_str());
	return;		return;
	}		}
	proto.flush(fd.get());		proto.flush(fd.get());
	}		}

			void StatsLogProcessor::WriteDataToDiskLocked() {
			for (auto& pair : mMetricsManagers) {
			WriteDataToDiskLocked(pair.first);
			}
			}

			void StatsLogProcessor::WriteDataToDisk() {
			std::lock_guard<std::mutex> lock(mMetricsMutex);
			WriteDataToDiskLocked();
	}		}

	} // namespace statsd		} // namespace statsd

cmds/statsd/src/StatsLogProcessor.h

+6 −1

Original line number	Original line	Diff line number	Diff line
	@@ -48,7 +48,8 @@ public:

	size_t GetMetricsSize(const ConfigKey& key) const;		size_t GetMetricsSize(const ConfigKey& key) const;

	void onDumpReport(const ConfigKey& key, const int64_t dumpTimeNs, vector<uint8_t>* outData);		void onDumpReport(const ConfigKey& key, const int64_t dumpTimeNs,
			const bool include_current_partial_bucket, vector<uint8_t>* outData);

	/* Tells MetricsManager that the alarms in alarmSet have fired. Modifies anomaly alarmSet. */		/* Tells MetricsManager that the alarms in alarmSet have fired. Modifies anomaly alarmSet. */
	void onAnomalyAlarmFired(		void onAnomalyAlarmFired(
	@@ -102,7 +103,11 @@ private:
	void OnConfigUpdatedLocked(		void OnConfigUpdatedLocked(
	const int64_t currentTimestampNs, const ConfigKey& key, const StatsdConfig& config);		const int64_t currentTimestampNs, const ConfigKey& key, const StatsdConfig& config);

			void WriteDataToDiskLocked();
			void WriteDataToDiskLocked(const ConfigKey& key);

	void onConfigMetricsReportLocked(const ConfigKey& key, const int64_t dumpTimeStampNs,		void onConfigMetricsReportLocked(const ConfigKey& key, const int64_t dumpTimeStampNs,
			const bool include_current_partial_bucket,
	util::ProtoOutputStream* proto);		util::ProtoOutputStream* proto);

	/* Check if we should send a broadcast if approaching memory limits and if we're over, we		/* Check if we should send a broadcast if approaching memory limits and if we're over, we

cmds/statsd/src/StatsService.cpp

+3 −2

Original line number	Original line	Diff line number	Diff line
	@@ -501,7 +501,7 @@ status_t StatsService::cmd_dump_report(FILE* out, FILE* err, const Vector<String
	if (good) {		if (good) {
	vector<uint8_t> data;		vector<uint8_t> data;
	mProcessor->onDumpReport(ConfigKey(uid, StrToInt64(name)), getElapsedRealtimeNs(),		mProcessor->onDumpReport(ConfigKey(uid, StrToInt64(name)), getElapsedRealtimeNs(),
	&data);		false /* include_current_bucket*/, &data);
	// TODO: print the returned StatsLogReport to file instead of printing to logcat.		// TODO: print the returned StatsLogReport to file instead of printing to logcat.
	if (proto) {		if (proto) {
	for (size_t i = 0; i < data.size(); i ++) {		for (size_t i = 0; i < data.size(); i ++) {
	@@ -800,7 +800,8 @@ Status StatsService::getData(int64_t key, vector<uint8_t>* output) {
	VLOG("StatsService::getData with Pid %i, Uid %i", ipc->getCallingPid(), ipc->getCallingUid());		VLOG("StatsService::getData with Pid %i, Uid %i", ipc->getCallingPid(), ipc->getCallingUid());
	if (checkCallingPermission(String16(kPermissionDump))) {		if (checkCallingPermission(String16(kPermissionDump))) {
	ConfigKey configKey(ipc->getCallingUid(), key);		ConfigKey configKey(ipc->getCallingUid(), key);
	mProcessor->onDumpReport(configKey, getElapsedRealtimeNs(), output);		mProcessor->onDumpReport(configKey, getElapsedRealtimeNs(),
			false /* include_current_bucket*/, output);
	return Status::ok();		return Status::ok();
	} else {		} else {
	return Status::fromExceptionCode(binder::Status::EX_SECURITY);		return Status::fromExceptionCode(binder::Status::EX_SECURITY);

cmds/statsd/src/guardrail/StatsdStats.cpp

+22 −13

Original line number	Original line	Diff line number	Diff line
	@@ -76,7 +76,8 @@ const int FIELD_ID_CONFIG_STATS_ALERT_COUNT = 8;
	const int FIELD_ID_CONFIG_STATS_VALID = 9;		const int FIELD_ID_CONFIG_STATS_VALID = 9;
	const int FIELD_ID_CONFIG_STATS_BROADCAST = 10;		const int FIELD_ID_CONFIG_STATS_BROADCAST = 10;
	const int FIELD_ID_CONFIG_STATS_DATA_DROP = 11;		const int FIELD_ID_CONFIG_STATS_DATA_DROP = 11;
	const int FIELD_ID_CONFIG_STATS_DUMP_REPORT = 12;		const int FIELD_ID_CONFIG_STATS_DUMP_REPORT_TIME = 12;
			const int FIELD_ID_CONFIG_STATS_DUMP_REPORT_BYTES = 20;
	const int FIELD_ID_CONFIG_STATS_MATCHER_STATS = 13;		const int FIELD_ID_CONFIG_STATS_MATCHER_STATS = 13;
	const int FIELD_ID_CONFIG_STATS_CONDITION_STATS = 14;		const int FIELD_ID_CONFIG_STATS_CONDITION_STATS = 14;
	const int FIELD_ID_CONFIG_STATS_METRIC_STATS = 15;		const int FIELD_ID_CONFIG_STATS_METRIC_STATS = 15;
	@@ -215,21 +216,22 @@ void StatsdStats::noteDataDropped(const ConfigKey& key, int32_t timeSec) {
	it->second->data_drop_time_sec.push_back(timeSec);		it->second->data_drop_time_sec.push_back(timeSec);
	}		}

	void StatsdStats::noteMetricsReportSent(const ConfigKey& key) {		void StatsdStats::noteMetricsReportSent(const ConfigKey& key, const size_t num_bytes) {
	noteMetricsReportSent(key, getWallClockSec());		noteMetricsReportSent(key, num_bytes, getWallClockSec());
	}		}

	void StatsdStats::noteMetricsReportSent(const ConfigKey& key, int32_t timeSec) {		void StatsdStats::noteMetricsReportSent(const ConfigKey& key, const size_t num_bytes,
			int32_t timeSec) {
	lock_guard<std::mutex> lock(mLock);		lock_guard<std::mutex> lock(mLock);
	auto it = mConfigStats.find(key);		auto it = mConfigStats.find(key);
	if (it == mConfigStats.end()) {		if (it == mConfigStats.end()) {
	ALOGE("Config key %s not found!", key.ToString().c_str());		ALOGE("Config key %s not found!", key.ToString().c_str());
	return;		return;
	}		}
	if (it->second->dump_report_time_sec.size() == kMaxTimestampCount) {		if (it->second->dump_report_stats.size() == kMaxTimestampCount) {
	it->second->dump_report_time_sec.pop_front();		it->second->dump_report_stats.pop_front();
	}		}
	it->second->dump_report_time_sec.push_back(timeSec);		it->second->dump_report_stats.push_back(std::make_pair(timeSec, num_bytes));
	}		}

	void StatsdStats::noteUidMapDropped(int deltas) {		void StatsdStats::noteUidMapDropped(int deltas) {
	@@ -383,7 +385,7 @@ void StatsdStats::resetInternalLocked() {
	for (auto& config : mConfigStats) {		for (auto& config : mConfigStats) {
	config.second->broadcast_sent_time_sec.clear();		config.second->broadcast_sent_time_sec.clear();
	config.second->data_drop_time_sec.clear();		config.second->data_drop_time_sec.clear();
	config.second->dump_report_time_sec.clear();		config.second->dump_report_stats.clear();
	config.second->annotations.clear();		config.second->annotations.clear();
	config.second->matcher_stats.clear();		config.second->matcher_stats.clear();
	config.second->condition_stats.clear();		config.second->condition_stats.clear();
	@@ -442,8 +444,8 @@ void StatsdStats::dumpStats(FILE* out) const {
	fprintf(out, "\tdata drop time: %d\n", dataDropTime);		fprintf(out, "\tdata drop time: %d\n", dataDropTime);
	}		}

	for (const auto& dumpTime : configStats->dump_report_time_sec) {		for (const auto& dump : configStats->dump_report_stats) {
	fprintf(out, "\tdump report time: %d\n", dumpTime);		fprintf(out, "\tdump report time: %d bytes: %lld\n", dump.first, (long long)dump.second);
	}		}

	for (const auto& stats : pair.second->matcher_stats) {		for (const auto& stats : pair.second->matcher_stats) {
	@@ -536,9 +538,16 @@ void addConfigStatsToProto(const ConfigStats& configStats, ProtoOutputStream* pr
	drop);		drop);
	}		}

	for (const auto& dump : configStats.dump_report_time_sec) {		for (const auto& dump : configStats.dump_report_stats) {
	proto->write(FIELD_TYPE_INT32 \| FIELD_ID_CONFIG_STATS_DUMP_REPORT \| FIELD_COUNT_REPEATED,		proto->write(FIELD_TYPE_INT32 \| FIELD_ID_CONFIG_STATS_DUMP_REPORT_TIME \|
	dump);		FIELD_COUNT_REPEATED,
			dump.first);
			}

			for (const auto& dump : configStats.dump_report_stats) {
			proto->write(FIELD_TYPE_INT64 \| FIELD_ID_CONFIG_STATS_DUMP_REPORT_BYTES \|
			FIELD_COUNT_REPEATED,
			(long long)dump.second);
	}		}

	for (const auto& annotation : configStats.annotations) {		for (const auto& annotation : configStats.annotations) {

cmds/statsd/src/guardrail/StatsdStats.h

+3 −3

Original line number	Original line	Diff line number	Diff line
	@@ -43,7 +43,7 @@ struct ConfigStats {

	std::list<int32_t> broadcast_sent_time_sec;		std::list<int32_t> broadcast_sent_time_sec;
	std::list<int32_t> data_drop_time_sec;		std::list<int32_t> data_drop_time_sec;
	std::list<int32_t> dump_report_time_sec;		std::list<std::pair<int32_t, int64_t>> dump_report_stats;

	// Stores how many times a matcher have been matched. The map size is capped by kMaxConfigCount.		// Stores how many times a matcher have been matched. The map size is capped by kMaxConfigCount.
	std::map<const int64_t, int> matcher_stats;		std::map<const int64_t, int> matcher_stats;
	@@ -177,7 +177,7 @@ public:
	*		*
	* The report may be requested via StatsManager API, or through adb cmd.		* The report may be requested via StatsManager API, or through adb cmd.
	*/		*/
	void noteMetricsReportSent(const ConfigKey& key);		void noteMetricsReportSent(const ConfigKey& key, const size_t num_bytes);

	/**		/**
	* Report the size of output tuple of a condition.		* Report the size of output tuple of a condition.
	@@ -355,7 +355,7 @@ private:

	void noteDataDropped(const ConfigKey& key, int32_t timeSec);		void noteDataDropped(const ConfigKey& key, int32_t timeSec);

	void noteMetricsReportSent(const ConfigKey& key, int32_t timeSec);		void noteMetricsReportSent(const ConfigKey& key, const size_t num_bytes, int32_t timeSec);

	void noteBroadcastSent(const ConfigKey& key, int32_t timeSec);		void noteBroadcastSent(const ConfigKey& key, int32_t timeSec);