Merge "logd: optionally track the full size of log buffers" am: 478d93846d (a7aaf84b) · Commits · e / os / android_system_core

logd/LogBufferElement.cpp

+5 −0

Original line number	Diff line number	Diff line
		@@ -99,6 +99,10 @@ uint32_t LogBufferElement::GetTag() const {
		}

		LogStatisticsElement LogBufferElement::ToLogStatisticsElement() const {
		// Estimate the size of this element in the parent std::list<> by adding two void*'s
		// corresponding to the next/prev pointers and aligning to 64 bit.
		uint16_t element_in_list_size =
		(sizeof(this) + 2 sizeof(void*) + sizeof(uint64_t) - 1) & -sizeof(uint64_t);
		return LogStatisticsElement{
		.uid = uid(),
		.pid = pid(),
		@@ -109,6 +113,7 @@ LogStatisticsElement LogBufferElement::ToLogStatisticsElement() const {
		.msg_len = msg_len(),
		.dropped_count = dropped_count(),
		.log_id = log_id(),
		.total_len = static_cast<uint16_t>(element_in_list_size + msg_len()),
		};
		}

logd/LogBufferTest.h

+1 −1

Original line number	Diff line number	Diff line
		@@ -87,6 +87,6 @@ class LogBufferTest : public testing::TestWithParam<std::string> {
		LogReaderList reader_list_;
		LogTags tags_;
		PruneList prune_;
		LogStatistics stats_{false};
		LogStatistics stats_{false, true};
		std::unique_ptr<LogBuffer> log_buffer_;
		};

logd/LogStatistics.cpp

+71 −12

Original line number	Diff line number	Diff line
		@@ -27,6 +27,7 @@

		#include <list>

		#include <android-base/logging.h>
		#include <private/android_logger.h>

		#include "LogBufferElement.h"
		@@ -60,7 +61,8 @@ static std::string TagNameKey(const LogStatisticsElement& element) {
		return std::string(msg, len);
		}

		LogStatistics::LogStatistics(bool enable_statistics) : enable(enable_statistics) {
		LogStatistics::LogStatistics(bool enable_statistics, bool track_total_size)
		: enable(enable_statistics), track_total_size_(track_total_size) {
		log_time now(CLOCK_REALTIME);
		log_id_for_each(id) {
		mSizes[id] = 0;
		@@ -113,10 +115,15 @@ void LogStatistics::AddTotal(log_id_t log_id, uint16_t size) {
		++mElementsTotal[log_id];
		}

		void LogStatistics::Add(const LogStatisticsElement& element) {
		void LogStatistics::Add(LogStatisticsElement element) {
		auto lock = std::lock_guard{lock_};

		if (!track_total_size_) {
		element.total_len = element.msg_len;
		}

		log_id_t log_id = element.log_id;
		uint16_t size = element.msg_len;
		uint16_t size = element.total_len;
		mSizes[log_id] += size;
		++mElements[log_id];

		@@ -184,10 +191,15 @@ void LogStatistics::Add(const LogStatisticsElement& element) {
		}
		}

		void LogStatistics::Subtract(const LogStatisticsElement& element) {
		void LogStatistics::Subtract(LogStatisticsElement element) {
		auto lock = std::lock_guard{lock_};

		if (!track_total_size_) {
		element.total_len = element.msg_len;
		}

		log_id_t log_id = element.log_id;
		uint16_t size = element.msg_len;
		uint16_t size = element.total_len;
		mSizes[log_id] -= size;
		--mElements[log_id];
		if (element.dropped_count) {
		@@ -230,7 +242,9 @@ void LogStatistics::Subtract(const LogStatisticsElement& element) {

		// Atomically set an entry to drop
		// entry->setDropped(1) must follow this call, caller should do this explicitly.
		void LogStatistics::Drop(const LogStatisticsElement& element) {
		void LogStatistics::Drop(LogStatisticsElement element) {
		CHECK_EQ(element.dropped_count, 0U);

		auto lock = std::lock_guard{lock_};
		log_id_t log_id = element.log_id;
		uint16_t size = element.msg_len;
		@@ -265,6 +279,43 @@ void LogStatistics::Drop(const LogStatisticsElement& element) {
		tagNameTable.Subtract(TagNameKey(element), element);
		}

		void LogStatistics::Erase(LogStatisticsElement element) {
		CHECK_GT(element.dropped_count, 0U);
		CHECK_EQ(element.msg_len, 0U);

		auto lock = std::lock_guard{lock_};

		if (!track_total_size_) {
		element.total_len = 0;
		}

		log_id_t log_id = element.log_id;
		--mElements[log_id];
		--mDroppedElements[log_id];
		mSizes[log_id] -= element.total_len;

		uidTable[log_id].Erase(element.uid, element);
		if (element.uid == AID_SYSTEM) {
		pidSystemTable[log_id].Erase(element.pid, element);
		}

		if (!enable) {
		return;
		}

		pidTable.Erase(element.pid, element);
		tidTable.Erase(element.tid, element);

		uint32_t tag = element.tag;
		if (tag) {
		if (log_id == LOG_ID_SECURITY) {
		securityTagTable.Erase(tag, element);
		} else {
		tagTable.Erase(tag, element);
		}
		}
		}

		const char* LogStatistics::UidToName(uid_t uid) const {
		auto lock = std::lock_guard{lock_};
		return UidToNameLocked(uid);
		@@ -876,12 +927,20 @@ std::string LogStatistics::Format(uid_t uid, pid_t pid, unsigned int logMask) co
		if (els) {
		oldLength = output.length();
		if (spaces < 0) spaces = 0;
		// estimate the std::list overhead.
		size_t szs = 0;
		if (overhead_[id]) {
		szs = *overhead_[id];
		} else if (track_total_size_) {
		szs = mSizes[id];
		} else {
		// Legacy fallback for Chatty without track_total_size_
		// Estimate the size of this element in the parent std::list<> by adding two void*'s
		// corresponding to the next/prev pointers and aligning to 64 bit.
		static const size_t overhead =
		((sizeof(LogBufferElement) + sizeof(uint64_t) - 1) &
		-sizeof(uint64_t)) +
		sizeof(std::list<LogBufferElement*>);
		size_t szs = mSizes[id] + els * overhead;
		(sizeof(LogBufferElement) + 2 * sizeof(void*) + sizeof(uint64_t) - 1) &
		-sizeof(uint64_t);
		szs = mSizes[id] + els * overhead;
		}
		totalSize += szs;
		output += android::base::StringPrintf("%*s%zu", spaces, "", szs);
		spaces -= output.length() - oldLength;

logd/LogStatistics.h

+42 −18

Original line number	Diff line number	Diff line
		@@ -57,6 +57,7 @@ struct LogStatisticsElement {
		uint16_t msg_len;
		uint16_t dropped_count;
		log_id_t log_id;
		uint16_t total_len;
		};

		template <typename TKey, typename TEntry>
		@@ -172,6 +173,13 @@ class LogHashtable {
		}
		}

		void Erase(const TKey& key, const LogStatisticsElement& element) {
		iterator it = map.find(key);
		if (it != map.end()) {
		it->second.Erase(element);
		}
		}

		iterator begin() { return map.begin(); }
		const_iterator begin() const { return map.begin(); }
		iterator end() { return map.end(); }
		@@ -181,15 +189,17 @@ class LogHashtable {
		class EntryBase {
		public:
		EntryBase() : size_(0) {}
		explicit EntryBase(const LogStatisticsElement& element) : size_(element.msg_len) {}
		explicit EntryBase(const LogStatisticsElement& element) : size_(element.total_len) {}

		size_t getSizes() const { return size_; }

		void Add(const LogStatisticsElement& element) { size_ += element.msg_len; }
		void Add(const LogStatisticsElement& element) { size_ += element.total_len; }
		bool Subtract(const LogStatisticsElement& element) {
		size_ -= element.msg_len;
		return !size_;
		size_ -= element.total_len;
		return size_ == 0;
		}
		void Drop(const LogStatisticsElement& element) { size_ -= element.msg_len; }
		void Erase(const LogStatisticsElement& element) { size_ -= element.total_len; }

		static constexpr size_t PRUNED_LEN = 14;
		static constexpr size_t TOTAL_LEN = 80;
		@@ -229,11 +239,11 @@ class EntryBaseDropped : public EntryBase {
		}
		bool Subtract(const LogStatisticsElement& element) {
		dropped_ -= element.dropped_count;
		return EntryBase::Subtract(element) && !dropped_;
		return EntryBase::Subtract(element) && dropped_ == 0;
		}
		void Drop(const LogStatisticsElement& element) {
		dropped_ += 1;
		EntryBase::Subtract(element);
		EntryBase::Drop(element);
		}

		private:
		@@ -505,20 +515,23 @@ class LogStatistics {
		}

		public:
		explicit LogStatistics(bool enable_statistics);
		LogStatistics(bool enable_statistics, bool track_total_size);

		void AddTotal(log_id_t log_id, uint16_t size) EXCLUDES(lock_);
		void Add(const LogStatisticsElement& entry) EXCLUDES(lock_);
		void Subtract(const LogStatisticsElement& entry) EXCLUDES(lock_);
		// entry->setDropped(1) must follow this call
		void Drop(const LogStatisticsElement& entry) EXCLUDES(lock_);
		// Correct for coalescing two entries referencing dropped content
		void Erase(const LogStatisticsElement& element) EXCLUDES(lock_) {
		auto lock = std::lock_guard{lock_};
		log_id_t log_id = element.log_id;
		--mElements[log_id];
		--mDroppedElements[log_id];
		}

		// Add is for adding an element to the log buffer. It may be a chatty element in the case of
		// log deduplication. Add the total size of the element to statistics.
		void Add(LogStatisticsElement entry) EXCLUDES(lock_);
		// Subtract is for removing an element from the log buffer. It may be a chatty element.
		// Subtract the total size of the element from statistics.
		void Subtract(LogStatisticsElement entry) EXCLUDES(lock_);
		// Drop is for converting a normal element into a chatty element. entry->setDropped(1) must
		// follow this call. Subtract only msg_len from statistics, since a chatty element will remain.
		void Drop(LogStatisticsElement entry) EXCLUDES(lock_);
		// Erase is for coalescing two chatty elements into one. Erase() is called on the element that
		// is removed from the log buffer. Subtract the total size of the element, which is by
		// definition only the size of the LogBufferElement + list overhead for chatty elements.
		void Erase(LogStatisticsElement element) EXCLUDES(lock_);

		void WorstTwoUids(log_id id, size_t threshold, int* worst, size_t* worst_sizes,
		size_t* second_worst_sizes) const EXCLUDES(lock_);
		@@ -533,6 +546,9 @@ class LogStatistics {
		// Snapshot of the sizes for a given log buffer.
		size_t Sizes(log_id_t id) const EXCLUDES(lock_) {
		auto lock = std::lock_guard{lock_};
		if (overhead_[id]) {
		return *overhead_[id];
		}
		return mSizes[id];
		}
		// TODO: Get rid of this entirely.
		@@ -546,6 +562,11 @@ class LogStatistics {
		uid_t PidToUid(pid_t pid) EXCLUDES(lock_);
		const char* UidToName(uid_t uid) const EXCLUDES(lock_);

		void set_overhead(log_id_t id, size_t size) {
		auto lock = std::lock_guard{lock_};
		overhead_[id] = size;
		}

		private:
		template <typename TKey, typename TEntry>
		void WorstTwoWithThreshold(const LogHashtable<TKey, TEntry>& table, size_t threshold,
		@@ -559,4 +580,7 @@ class LogStatistics {
		const char* UidToNameLocked(uid_t uid) const REQUIRES(lock_);

		mutable std::mutex lock_;
		bool track_total_size_;

		std::optional<size_t> overhead_[LOG_ID_MAX] GUARDED_BY(lock_);
		};

logd/SerializedLogBuffer.cpp

+16 −16

Original line number	Diff line number	Diff line
		@@ -109,19 +109,16 @@ int SerializedLogBuffer::Log(log_id_t log_id, log_time realtime, uid_t uid, pid_
		}

		void SerializedLogBuffer::MaybePrune(log_id_t log_id) {
		auto get_total_size = [](const auto& buffer) {
		size_t total_size = 0;
		for (const auto& chunk : buffer) {
		total_size += chunk.PruneSize();
		}
		return total_size;
		};
		size_t total_size = get_total_size(logs_[log_id]);
		size_t total_size = GetSizeUsed(log_id);
		size_t after_size = total_size;
		if (total_size > max_size_[log_id]) {
		Prune(log_id, total_size - max_size_[log_id], 0);
		after_size = GetSizeUsed(log_id);
		LOG(INFO) << "Pruned Logs from log_id: " << log_id << ", previous size: " << total_size
		<< " after size: " << get_total_size(logs_[log_id]);
		<< " after size: " << after_size;
		}

		stats_->set_overhead(log_id, after_size);
		}

		void SerializedLogBuffer::StartDeleterThread() {
		@@ -349,19 +346,22 @@ bool SerializedLogBuffer::Clear(log_id_t id, uid_t uid) {
		return Prune(id, ULONG_MAX, uid);
		}

		unsigned long SerializedLogBuffer::GetSizeUsed(log_id_t id) {
		size_t total_size = 0;
		for (const auto& chunk : logs_[id]) {
		total_size += chunk.PruneSize();
		}
		return total_size;
		}

		unsigned long SerializedLogBuffer::GetSize(log_id_t id) {
		auto lock = std::lock_guard{lock_};
		size_t retval = 2 * max_size_[id] / 3; // See the comment in SetSize().
		return retval;
		return max_size_[id];
		}

		// New SerializedLogChunk objects will be allocated according to the new size, but older one are
		// unchanged. MaybePrune() is called on the log buffer to reduce it to an appropriate size if the
		// new size is lower.
		// ChattyLogBuffer/SimpleLogBuffer don't consider the 'Overhead' of LogBufferElement or the
		// std::list<> overhead as part of the log size. SerializedLogBuffer does by its very nature, so
		// the 'size' metric is not compatible between them. Their actual memory usage is between 1.5x and
		// 2x of what they claim to use, so we conservatively set our internal size as size + size / 2.
		int SerializedLogBuffer::SetSize(log_id_t id, unsigned long size) {
		// Reasonable limits ...
		if (!__android_logger_valid_buffer_size(size)) {
		@@ -369,7 +369,7 @@ int SerializedLogBuffer::SetSize(log_id_t id, unsigned long size) {
		}

		auto lock = std::lock_guard{lock_};
		max_size_[id] = size + size / 2;
		max_size_[id] = size;

		MaybePrune(id);