Merge changes I724584dd,I337beaf0,Ica1e9ca6,Id278f467,I54015548 into udc-dev am: e055650d81 (449669f0) · Commits · e / os / android_frameworks_native

opengl/libs/Android.bp

+1 −0

Original line number	Original line	Diff line number	Diff line
	@@ -205,6 +205,7 @@ cc_test {
	srcs: [		srcs: [
	"EGL/BlobCache.cpp",		"EGL/BlobCache.cpp",
	"EGL/BlobCache_test.cpp",		"EGL/BlobCache_test.cpp",
			"EGL/FileBlobCache.cpp",
	"EGL/MultifileBlobCache.cpp",		"EGL/MultifileBlobCache.cpp",
	"EGL/MultifileBlobCache_test.cpp",		"EGL/MultifileBlobCache_test.cpp",
	],		],

opengl/libs/EGL/FileBlobCache.cpp

+1 −1

Original line number	Original line	Diff line number	Diff line
	@@ -31,7 +31,7 @@ static const size_t cacheFileHeaderSize = 8;

	namespace android {		namespace android {

	static uint32_t crc32c(const uint8_t* buf, size_t len) {		uint32_t crc32c(const uint8_t* buf, size_t len) {
	const uint32_t polyBits = 0x82F63B78;		const uint32_t polyBits = 0x82F63B78;
	uint32_t r = 0;		uint32_t r = 0;
	for (size_t i = 0; i < len; i++) {		for (size_t i = 0; i < len; i++) {

opengl/libs/EGL/FileBlobCache.h

+2 −0

Original line number	Original line	Diff line number	Diff line
	@@ -22,6 +22,8 @@

	namespace android {		namespace android {

			uint32_t crc32c(const uint8_t* buf, size_t len);

	class FileBlobCache : public BlobCache {		class FileBlobCache : public BlobCache {
	public:		public:
	// FileBlobCache attempts to load the saved cache contents from disk into		// FileBlobCache attempts to load the saved cache contents from disk into

opengl/libs/EGL/MultifileBlobCache.cpp

+114 −68

Original line number	Original line	Diff line number	Diff line
	@@ -39,21 +39,10 @@

	using namespace std::literals;		using namespace std::literals;

	namespace {		constexpr uint32_t kMultifileMagic = 'MFB$';
			constexpr uint32_t kCrcPlaceholder = 0;
	// Open the file and determine the size of the value it contains
	size_t getValueSizeFromFile(int fd, const std::string& entryPath) {
	// Read the beginning of the file to get header
	android::MultifileHeader header;
	size_t result = read(fd, static_cast<void*>(&header), sizeof(android::MultifileHeader));
	if (result != sizeof(android::MultifileHeader)) {
	ALOGE("Error reading MultifileHeader from cache entry (%s): %s", entryPath.c_str(),
	std::strerror(errno));
	return 0;
	}

	return header.valueSize;		namespace {
	}

	// Helper function to close entries or free them		// Helper function to close entries or free them
	void freeHotCacheEntry(android::MultifileHotCache& entry) {		void freeHotCacheEntry(android::MultifileHotCache& entry) {
	@@ -73,12 +62,14 @@ void freeHotCacheEntry(android::MultifileHotCache& entry) {

	namespace android {		namespace android {

	MultifileBlobCache::MultifileBlobCache(size_t maxTotalSize, size_t maxHotCacheSize,		MultifileBlobCache::MultifileBlobCache(size_t maxKeySize, size_t maxValueSize, size_t maxTotalSize,
	const std::string& baseDir)		const std::string& baseDir)
	: mInitialized(false),		: mInitialized(false),
			mMaxKeySize(maxKeySize),
			mMaxValueSize(maxValueSize),
	mMaxTotalSize(maxTotalSize),		mMaxTotalSize(maxTotalSize),
	mTotalCacheSize(0),		mTotalCacheSize(0),
	mHotCacheLimit(maxHotCacheSize),		mHotCacheLimit(0),
	mHotCacheSize(0),		mHotCacheSize(0),
	mWorkerThreadIdle(true) {		mWorkerThreadIdle(true) {
	if (baseDir.empty()) {		if (baseDir.empty()) {
	@@ -89,9 +80,9 @@ MultifileBlobCache::MultifileBlobCache(size_t maxTotalSize, size_t maxHotCacheSi
	// Establish the name of our multifile directory		// Establish the name of our multifile directory
	mMultifileDirName = baseDir + ".multifile";		mMultifileDirName = baseDir + ".multifile";

	// Set a limit for max key and value, ensuring at least one entry can always fit in hot cache		// Set the hotcache limit to be large enough to contain one max entry
	mMaxKeySize = mHotCacheLimit / 4;		// This ensure the hot cache is always large enough for single entry
	mMaxValueSize = mHotCacheLimit / 2;		mHotCacheLimit = mMaxKeySize + mMaxValueSize + sizeof(MultifileHeader);

	ALOGV("INIT: Initializing multifile blobcache with maxKeySize=%zu and maxValueSize=%zu",		ALOGV("INIT: Initializing multifile blobcache with maxKeySize=%zu and maxValueSize=%zu",
	mMaxKeySize, mMaxValueSize);		mMaxKeySize, mMaxValueSize);
	@@ -129,6 +120,15 @@ MultifileBlobCache::MultifileBlobCache(size_t maxTotalSize, size_t maxHotCacheSi
	return;		return;
	}		}

			// If the cache entry is damaged or no good, remove it
			if (st.st_size <= 0 \|\| st.st_atime <= 0) {
			ALOGE("INIT: Entry %u has invalid stats! Removing.", entryHash);
			if (remove(fullPath.c_str()) != 0) {
			ALOGE("Error removing %s: %s", fullPath.c_str(), std::strerror(errno));
			}
			continue;
			}

	// Open the file so we can read its header		// Open the file so we can read its header
	int fd = open(fullPath.c_str(), O_RDONLY);		int fd = open(fullPath.c_str(), O_RDONLY);
	if (fd == -1) {		if (fd == -1) {
	@@ -137,40 +137,67 @@ MultifileBlobCache::MultifileBlobCache(size_t maxTotalSize, size_t maxHotCacheSi
	return;		return;
	}		}

	// Look up the details we track about each file		// Read the beginning of the file to get header
	size_t valueSize = getValueSizeFromFile(fd, fullPath);		MultifileHeader header;
			size_t result = read(fd, static_cast<void*>(&header), sizeof(MultifileHeader));
			if (result != sizeof(MultifileHeader)) {
			ALOGE("Error reading MultifileHeader from cache entry (%s): %s",
			fullPath.c_str(), std::strerror(errno));
			return;
			}

	// If the cache entry is damaged or no good, remove it		// Verify header magic
	// TODO: Perform any other checks		if (header.magic != kMultifileMagic) {
	if (valueSize <= 0 \|\| st.st_size <= 0 \|\| st.st_atime <= 0) {		ALOGE("INIT: Entry %u has bad magic (%u)! Removing.", entryHash, header.magic);
	ALOGV("INIT: Entry %u has a problem! Removing.", entryHash);
	if (remove(fullPath.c_str()) != 0) {		if (remove(fullPath.c_str()) != 0) {
	ALOGE("Error removing %s: %s", fullPath.c_str(), std::strerror(errno));		ALOGE("Error removing %s: %s", fullPath.c_str(), std::strerror(errno));
	}		}
	continue;		continue;
	}		}

	ALOGV("INIT: Entry %u is good, tracking it now.", entryHash);

	// Note: Converting from off_t (signed) to size_t (unsigned)		// Note: Converting from off_t (signed) to size_t (unsigned)
	size_t fileSize = static_cast<size_t>(st.st_size);		size_t fileSize = static_cast<size_t>(st.st_size);
	time_t accessTime = st.st_atime;

	// Track details for rapid lookup later
	trackEntry(entryHash, valueSize, fileSize, accessTime);

	// Track the total size
	increaseTotalCacheSize(fileSize);

	// Preload the entry for fast retrieval
	if ((mHotCacheSize + fileSize) < mHotCacheLimit) {
	// Memory map the file		// Memory map the file
	uint8_t* mappedEntry = reinterpret_cast<uint8_t*>(		uint8_t* mappedEntry = reinterpret_cast<uint8_t*>(
	mmap(nullptr, fileSize, PROT_READ, MAP_PRIVATE, fd, 0));		mmap(nullptr, fileSize, PROT_READ, MAP_PRIVATE, fd, 0));
	if (mappedEntry == MAP_FAILED) {		if (mappedEntry == MAP_FAILED) {
	ALOGE("Failed to mmap cacheEntry, error: %s", std::strerror(errno));		ALOGE("Failed to mmap cacheEntry, error: %s", std::strerror(errno));
			return;
			}

			// Ensure we have a good CRC
			if (header.crc !=
			crc32c(mappedEntry + sizeof(MultifileHeader),
			fileSize - sizeof(MultifileHeader))) {
			ALOGE("INIT: Entry %u failed CRC check! Removing.", entryHash);
			if (remove(fullPath.c_str()) != 0) {
			ALOGE("Error removing %s: %s", fullPath.c_str(), std::strerror(errno));
			}
			continue;
			}

			// If the cache entry is damaged or no good, remove it
			if (header.keySize <= 0 \|\| header.valueSize <= 0) {
			ALOGE("INIT: Entry %u has a bad header keySize (%lu) or valueSize (%lu), "
			"removing.",
			entryHash, header.keySize, header.valueSize);
			if (remove(fullPath.c_str()) != 0) {
			ALOGE("Error removing %s: %s", fullPath.c_str(), std::strerror(errno));
			}
			continue;
	}		}

			ALOGV("INIT: Entry %u is good, tracking it now.", entryHash);

			// Track details for rapid lookup later
			trackEntry(entryHash, header.valueSize, fileSize, st.st_atime);

			// Track the total size
			increaseTotalCacheSize(fileSize);

			// Preload the entry for fast retrieval
			if ((mHotCacheSize + fileSize) < mHotCacheLimit) {
	ALOGV("INIT: Populating hot cache with fd = %i, cacheEntry = %p for "		ALOGV("INIT: Populating hot cache with fd = %i, cacheEntry = %p for "
	"entryHash %u",		"entryHash %u",
	fd, mappedEntry, entryHash);		fd, mappedEntry, entryHash);
	@@ -183,6 +210,8 @@ MultifileBlobCache::MultifileBlobCache(size_t maxTotalSize, size_t maxHotCacheSi
	return;		return;
	}		}
	} else {		} else {
			// If we're not keeping it in hot cache, unmap it now
			munmap(mappedEntry, fileSize);
	close(fd);		close(fd);
	}		}
	}		}
	@@ -227,7 +256,7 @@ void MultifileBlobCache::set(const void* key, EGLsizeiANDROID keySize, const voi

	// Ensure key and value are under their limits		// Ensure key and value are under their limits
	if (keySize > mMaxKeySize \|\| valueSize > mMaxValueSize) {		if (keySize > mMaxKeySize \|\| valueSize > mMaxValueSize) {
	ALOGV("SET: keySize (%lu vs %zu) or valueSize (%lu vs %zu) too large", keySize, mMaxKeySize,		ALOGW("SET: keySize (%lu vs %zu) or valueSize (%lu vs %zu) too large", keySize, mMaxKeySize,
	valueSize, mMaxValueSize);		valueSize, mMaxValueSize);
	return;		return;
	}		}
	@@ -240,17 +269,18 @@ void MultifileBlobCache::set(const void* key, EGLsizeiANDROID keySize, const voi
	// If we're going to be over the cache limit, kick off a trim to clear space		// If we're going to be over the cache limit, kick off a trim to clear space
	if (getTotalSize() + fileSize > mMaxTotalSize) {		if (getTotalSize() + fileSize > mMaxTotalSize) {
	ALOGV("SET: Cache is full, calling trimCache to clear space");		ALOGV("SET: Cache is full, calling trimCache to clear space");
	trimCache(mMaxTotalSize);		trimCache();
	}		}

	ALOGV("SET: Add %u to cache", entryHash);		ALOGV("SET: Add %u to cache", entryHash);

	uint8_t* buffer = new uint8_t[fileSize];		uint8_t* buffer = new uint8_t[fileSize];

	// Write the key and value after the header		// Write placeholders for magic and CRC until deferred thread completes the write
	android::MultifileHeader header = {keySize, valueSize};		android::MultifileHeader header = {kMultifileMagic, kCrcPlaceholder, keySize, valueSize};
	memcpy(static_cast<void>(buffer), static_cast<const void>(&header),		memcpy(static_cast<void>(buffer), static_cast<const void>(&header),
	sizeof(android::MultifileHeader));		sizeof(android::MultifileHeader));
			// Write the key and value after the header
	memcpy(static_cast<void>(buffer + sizeof(MultifileHeader)), static_cast<const void>(key),		memcpy(static_cast<void>(buffer + sizeof(MultifileHeader)), static_cast<const void>(key),
	keySize);		keySize);
	memcpy(static_cast<void*>(buffer + sizeof(MultifileHeader) + keySize),		memcpy(static_cast<void*>(buffer + sizeof(MultifileHeader) + keySize),
	@@ -269,13 +299,18 @@ void MultifileBlobCache::set(const void* key, EGLsizeiANDROID keySize, const voi

	// Sending -1 as the fd indicates we don't have an fd for this		// Sending -1 as the fd indicates we don't have an fd for this
	if (!addToHotCache(entryHash, -1, buffer, fileSize)) {		if (!addToHotCache(entryHash, -1, buffer, fileSize)) {
	ALOGE("GET: Failed to add %u to hot cache", entryHash);		ALOGE("SET: Failed to add %u to hot cache", entryHash);
			delete[] buffer;
	return;		return;
	}		}

	// Track that we're creating a pending write for this entry		// Track that we're creating a pending write for this entry
	// Include the buffer to handle the case when multiple writes are pending for an entry		// Include the buffer to handle the case when multiple writes are pending for an entry
			{
			// Synchronize access to deferred write status
			std::lock_guard<std::mutex> lock(mDeferredWriteStatusMutex);
	mDeferredWrites.insert(std::make_pair(entryHash, buffer));		mDeferredWrites.insert(std::make_pair(entryHash, buffer));
			}

	// Create deferred task to write to storage		// Create deferred task to write to storage
	ALOGV("SET: Adding task to queue.");		ALOGV("SET: Adding task to queue.");
	@@ -293,7 +328,7 @@ EGLsizeiANDROID MultifileBlobCache::get(const void* key, EGLsizeiANDROID keySize

	// Ensure key and value are under their limits		// Ensure key and value are under their limits
	if (keySize > mMaxKeySize \|\| valueSize > mMaxValueSize) {		if (keySize > mMaxKeySize \|\| valueSize > mMaxValueSize) {
	ALOGV("GET: keySize (%lu vs %zu) or valueSize (%lu vs %zu) too large", keySize, mMaxKeySize,		ALOGW("GET: keySize (%lu vs %zu) or valueSize (%lu vs %zu) too large", keySize, mMaxKeySize,
	valueSize, mMaxValueSize);		valueSize, mMaxValueSize);
	return 0;		return 0;
	}		}
	@@ -342,8 +377,15 @@ EGLsizeiANDROID MultifileBlobCache::get(const void* key, EGLsizeiANDROID keySize
	} else {		} else {
	ALOGV("GET: HotCache MISS for entry: %u", entryHash);		ALOGV("GET: HotCache MISS for entry: %u", entryHash);

	if (mDeferredWrites.find(entryHash) != mDeferredWrites.end()) {
	// Wait for writes to complete if there is an outstanding write for this entry		// Wait for writes to complete if there is an outstanding write for this entry
			bool wait = false;
			{
			// Synchronize access to deferred write status
			std::lock_guard<std::mutex> lock(mDeferredWriteStatusMutex);
			wait = mDeferredWrites.find(entryHash) != mDeferredWrites.end();
			}

			if (wait) {
	ALOGV("GET: Waiting for write to complete for %u", entryHash);		ALOGV("GET: Waiting for write to complete for %u", entryHash);
	waitForWorkComplete();		waitForWorkComplete();
	}		}
	@@ -455,6 +497,7 @@ bool MultifileBlobCache::addToHotCache(uint32_t newEntryHash, int newFd, uint8_t
	mHotCacheSize, newEntrySize, mHotCacheLimit, newEntryHash);		mHotCacheSize, newEntrySize, mHotCacheLimit, newEntryHash);

	// Wait for all the files to complete writing so our hot cache is accurate		// Wait for all the files to complete writing so our hot cache is accurate
			ALOGV("HOTCACHE(ADD): Waiting for work to complete for %u", newEntryHash);
	waitForWorkComplete();		waitForWorkComplete();

	// Free up old entries until under the limit		// Free up old entries until under the limit
	@@ -491,6 +534,7 @@ bool MultifileBlobCache::removeFromHotCache(uint32_t entryHash) {
	ALOGV("HOTCACHE(REMOVE): Removing %u from hot cache", entryHash);		ALOGV("HOTCACHE(REMOVE): Removing %u from hot cache", entryHash);

	// Wait for all the files to complete writing so our hot cache is accurate		// Wait for all the files to complete writing so our hot cache is accurate
			ALOGV("HOTCACHE(REMOVE): Waiting for work to complete for %u", entryHash);
	waitForWorkComplete();		waitForWorkComplete();

	ALOGV("HOTCACHE(REMOVE): Closing hot cache entry for %u", entryHash);		ALOGV("HOTCACHE(REMOVE): Closing hot cache entry for %u", entryHash);
	@@ -547,7 +591,7 @@ bool MultifileBlobCache::applyLRU(size_t cacheLimit) {
	}		}
	}		}

	ALOGV("LRU: Cache is emptry");		ALOGV("LRU: Cache is empty");
	return false;		return false;
	}		}

	@@ -556,25 +600,17 @@ bool MultifileBlobCache::applyLRU(size_t cacheLimit) {
	constexpr uint32_t kCacheLimitDivisor = 2;		constexpr uint32_t kCacheLimitDivisor = 2;

	// Calculate the cache size and remove old entries until under the limit		// Calculate the cache size and remove old entries until under the limit
	void MultifileBlobCache::trimCache(size_t cacheByteLimit) {		void MultifileBlobCache::trimCache() {
	// Start with the value provided by egl_cache
	size_t limit = cacheByteLimit;

	// Wait for all deferred writes to complete		// Wait for all deferred writes to complete
			ALOGV("TRIM: Waiting for work to complete.");
	waitForWorkComplete();		waitForWorkComplete();

	size_t size = getTotalSize();		ALOGV("TRIM: Reducing multifile cache size to %zu", mMaxTotalSize / kCacheLimitDivisor);
			if (!applyLRU(mMaxTotalSize / kCacheLimitDivisor)) {
	// If size is larger than the threshold, remove files using LRU
	if (size > limit) {
	ALOGV("TRIM: Multifile cache size is larger than %zu, removing old entries",
	cacheByteLimit);
	if (!applyLRU(limit / kCacheLimitDivisor)) {
	ALOGE("Error when clearing multifile shader cache");		ALOGE("Error when clearing multifile shader cache");
	return;		return;
	}		}
	}		}
	}

	// This function performs a task. It only knows how to write files to disk,		// This function performs a task. It only knows how to write files to disk,
	// but it could be expanded if needed.		// but it could be expanded if needed.
	@@ -600,6 +636,11 @@ void MultifileBlobCache::processTask(DeferredTask& task) {

	ALOGV("DEFERRED: Opened fd %i from %s", fd, fullPath.c_str());		ALOGV("DEFERRED: Opened fd %i from %s", fd, fullPath.c_str());

			// Add CRC check to the header (always do this last!)
			MultifileHeader* header = reinterpret_cast<MultifileHeader*>(buffer);
			header->crc =
			crc32c(buffer + sizeof(MultifileHeader), bufferSize - sizeof(MultifileHeader));

	ssize_t result = write(fd, buffer, bufferSize);		ssize_t result = write(fd, buffer, bufferSize);
	if (result != bufferSize) {		if (result != bufferSize) {
	ALOGE("Error writing fileSize to cache entry (%s): %s", fullPath.c_str(),		ALOGE("Error writing fileSize to cache entry (%s): %s", fullPath.c_str(),
	@@ -612,15 +653,20 @@ void MultifileBlobCache::processTask(DeferredTask& task) {

	// Erase the entry from mDeferredWrites		// Erase the entry from mDeferredWrites
	// Since there could be multiple outstanding writes for an entry, find the matching one		// Since there could be multiple outstanding writes for an entry, find the matching one
			{
			// Synchronize access to deferred write status
			std::lock_guard<std::mutex> lock(mDeferredWriteStatusMutex);
	typedef std::multimap<uint32_t, uint8_t*>::iterator entryIter;		typedef std::multimap<uint32_t, uint8_t*>::iterator entryIter;
	std::pair<entryIter, entryIter> iterPair = mDeferredWrites.equal_range(entryHash);		std::pair<entryIter, entryIter> iterPair = mDeferredWrites.equal_range(entryHash);
	for (entryIter it = iterPair.first; it != iterPair.second; ++it) {		for (entryIter it = iterPair.first; it != iterPair.second; ++it) {
	if (it->second == buffer) {		if (it->second == buffer) {
	ALOGV("DEFERRED: Marking write complete for %u at %p", it->first, it->second);		ALOGV("DEFERRED: Marking write complete for %u at %p", it->first,
			it->second);
	mDeferredWrites.erase(it);		mDeferredWrites.erase(it);
	break;		break;
	}		}
	}		}
			}

	return;		return;
	}		}

opengl/libs/EGL/MultifileBlobCache.h

+10 −3

Original line number	Original line	Diff line number	Diff line
	@@ -20,6 +20,7 @@
	#include <EGL/egl.h>		#include <EGL/egl.h>
	#include <EGL/eglext.h>		#include <EGL/eglext.h>

			#include <android-base/thread_annotations.h>
	#include <future>		#include <future>
	#include <map>		#include <map>
	#include <queue>		#include <queue>
	@@ -28,9 +29,13 @@
	#include <unordered_map>		#include <unordered_map>
	#include <unordered_set>		#include <unordered_set>

			#include "FileBlobCache.h"

	namespace android {		namespace android {

	struct MultifileHeader {		struct MultifileHeader {
			uint32_t magic;
			uint32_t crc;
	EGLsizeiANDROID keySize;		EGLsizeiANDROID keySize;
	EGLsizeiANDROID valueSize;		EGLsizeiANDROID valueSize;
	};		};
	@@ -86,7 +91,8 @@ private:

	class MultifileBlobCache {		class MultifileBlobCache {
	public:		public:
	MultifileBlobCache(size_t maxTotalSize, size_t maxHotCacheSize, const std::string& baseDir);		MultifileBlobCache(size_t maxKeySize, size_t maxValueSize, size_t maxTotalSize,
			const std::string& baseDir);
	~MultifileBlobCache();		~MultifileBlobCache();

	void set(const void* key, EGLsizeiANDROID keySize, const void* value,		void set(const void* key, EGLsizeiANDROID keySize, const void* value,
	@@ -114,7 +120,7 @@ private:
	bool addToHotCache(uint32_t entryHash, int fd, uint8_t* entryBufer, size_t entrySize);		bool addToHotCache(uint32_t entryHash, int fd, uint8_t* entryBufer, size_t entrySize);
	bool removeFromHotCache(uint32_t entryHash);		bool removeFromHotCache(uint32_t entryHash);

	void trimCache(size_t cacheByteLimit);		void trimCache();
	bool applyLRU(size_t cacheLimit);		bool applyLRU(size_t cacheLimit);

	bool mInitialized;		bool mInitialized;
	@@ -135,7 +141,8 @@ private:
	// Below are the components used for deferred writes		// Below are the components used for deferred writes

	// Track whether we have pending writes for an entry		// Track whether we have pending writes for an entry
	std::multimap<uint32_t, uint8_t*> mDeferredWrites;		std::mutex mDeferredWriteStatusMutex;
			std::multimap<uint32_t, uint8_t*> mDeferredWrites GUARDED_BY(mDeferredWriteStatusMutex);

	// Functions to work through tasks in the queue		// Functions to work through tasks in the queue
	void processTasks();		void processTasks();