BlobCache: implement cache serialization (0e1bc174) · Commits · e / os / android_frameworks_native-old

include/utils/BlobCache.h

+84 −12

Original line number	Diff line number	Diff line
		@@ -19,6 +19,7 @@

		#include <stddef.h>

		#include <utils/Flattenable.h>
		#include <utils/RefBase.h>
		#include <utils/SortedVector.h>
		#include <utils/threads.h>
		@@ -28,10 +29,11 @@ namespace android {
		// A BlobCache is an in-memory cache for binary key/value pairs. A BlobCache
		// does NOT provide any thread-safety guarantees.
		//
		// The cache contents can be serialized to a file and reloaded in a subsequent
		// execution of the program. This serialization is non-portable and should only
		// be loaded by the device that generated it.
		class BlobCache : public RefBase {
		// The cache contents can be serialized to an in-memory buffer or mmap'd file
		// and then reloaded in a subsequent execution of the program. This
		// serialization is non-portable and the data should only be used by the device
		// that generated it.
		class BlobCache : public RefBase, public Flattenable {
		public:

		// Create an empty blob cache. The blob cache will cache key/value pairs
		@@ -58,14 +60,13 @@ public:
		void set(const void* key, size_t keySize, const void* value,
		size_t valueSize);

		// The get function retrieves from the cache the binary value associated
		// with a given binary key. If the key is present in the cache then the
		// length of the binary value associated with that key is returned. If the
		// value argument is non-NULL and the size of the cached value is less than
		// valueSize bytes then the cached value is copied into the buffer pointed
		// to by the value argument. If the key is not present in the cache then 0
		// is returned and the buffer pointed to by the value argument is not
		// modified.
		// get retrieves from the cache the binary value associated with a given
		// binary key. If the key is present in the cache then the length of the
		// binary value associated with that key is returned. If the value argument
		// is non-NULL and the size of the cached value is less than valueSize bytes
		// then the cached value is copied into the buffer pointed to by the value
		// argument. If the key is not present in the cache then 0 is returned and
		// the buffer pointed to by the value argument is not modified.
		//
		// Note that when calling get multiple times with the same key, the later
		// calls may fail, returning 0, even if earlier calls succeeded. The return
		@@ -77,6 +78,37 @@ public:
		// 0 <= valueSize
		size_t get(const void* key, size_t keySize, void* value, size_t valueSize);

		// getFlattenedSize returns the number of bytes needed to store the entire
		// serialized cache.
		virtual size_t getFlattenedSize() const;

		// getFdCount returns the number of file descriptors that will result from
		// flattening the cache. This will always return 0 so as to allow the
		// flattened cache to be saved to disk and then later restored.
		virtual size_t getFdCount() const;

		// flatten serializes the current contents of the cache into the memory
		// pointed to by 'buffer'. The serialized cache contents can later be
		// loaded into a BlobCache object using the unflatten method. The contents
		// of the BlobCache object will not be modified.
		//
		// Preconditions:
		// size >= this.getFlattenedSize()
		// count == 0
		virtual status_t flatten(void* buffer, size_t size, int fds[],
		size_t count) const;

		// unflatten replaces the contents of the cache with the serialized cache
		// contents in the memory pointed to by 'buffer'. The previous contents of
		// the BlobCache will be evicted from the cache. If an error occurs while
		// unflattening the serialized cache contents then the BlobCache will be
		// left in an empty state.
		//
		// Preconditions:
		// count == 0
		virtual status_t unflatten(void const* buffer, size_t size, int fds[],
		size_t count);

		private:
		// Copying is disallowed.
		BlobCache(const BlobCache&);
		@@ -144,6 +176,46 @@ private:
		sp<Blob> mValue;
		};

		// A Header is the header for the entire BlobCache serialization format. No
		// need to make this portable, so we simply write the struct out.
		struct Header {
		// mMagicNumber is the magic number that identifies the data as
		// serialized BlobCache contents. It must always contain 'Blb$'.
		uint32_t mMagicNumber;

		// mBlobCacheVersion is the serialization format version.
		uint32_t mBlobCacheVersion;

		// mDeviceVersion is the device-specific version of the cache. This can
		// be used to invalidate the cache.
		uint32_t mDeviceVersion;

		// mNumEntries is number of cache entries following the header in the
		// data.
		size_t mNumEntries;
		};

		// An EntryHeader is the header for a serialized cache entry. No need to
		// make this portable, so we simply write the struct out. Each EntryHeader
		// is followed imediately by the key data and then the value data.
		//
		// The beginning of each serialized EntryHeader is 4-byte aligned, so the
		// number of bytes that a serialized cache entry will occupy is:
		//
		// ((sizeof(EntryHeader) + keySize + valueSize) + 3) & ~3
		//
		struct EntryHeader {
		// mKeySize is the size of the entry key in bytes.
		size_t mKeySize;

		// mValueSize is the size of the entry value in bytes.
		size_t mValueSize;

		// mData contains both the key and value data for the cache entry. The
		// key comes first followed immediately by the value.
		uint8_t mData[];
		};

		// mMaxKeySize is the maximum key size that will be cached. Calls to
		// BlobCache::set with a keySize parameter larger than mMaxKeySize will
		// simply not add the key/value pair to the cache.

libs/utils/BlobCache.cpp

+138 −2

Original line number	Diff line number	Diff line
		@@ -21,10 +21,20 @@
		#include <string.h>

		#include <utils/BlobCache.h>
		#include <utils/Errors.h>
		#include <utils/Log.h>

		namespace android {

		// BlobCache::Header::mMagicNumber value
		static const uint32_t blobCacheMagic = '_Bb$';

		// BlobCache::Header::mBlobCacheVersion value
		static const uint32_t blobCacheVersion = 1;

		// BlobCache::Header::mDeviceVersion value
		static const uint32_t blobCacheDeviceVersion = 1;

		BlobCache::BlobCache(size_t maxKeySize, size_t maxValueSize, size_t maxTotalSize):
		mMaxKeySize(maxKeySize),
		mMaxValueSize(maxValueSize),
		@@ -71,7 +81,6 @@ void BlobCache::set(const void* key, size_t keySize, const void* value,
		CacheEntry dummyEntry(dummyKey, NULL);

		while (true) {

		ssize_t index = mCacheEntries.indexOf(dummyEntry);
		if (index < 0) {
		// Create a new cache entry.
		@@ -150,6 +159,133 @@ size_t BlobCache::get(const void* key, size_t keySize, void* value,
		return valueBlobSize;
		}

		static inline size_t align4(size_t size) {
		return (size + 3) & ~3;
		}

		size_t BlobCache::getFlattenedSize() const {
		size_t size = sizeof(Header);
		for (size_t i = 0; i < mCacheEntries.size(); i++) {
		const CacheEntry& e(mCacheEntries[i]);
		sp<Blob> keyBlob = e.getKey();
		sp<Blob> valueBlob = e.getValue();
		size = align4(size);
		size += sizeof(EntryHeader) + keyBlob->getSize() +
		valueBlob->getSize();
		}
		return size;
		}

		size_t BlobCache::getFdCount() const {
		return 0;
		}

		status_t BlobCache::flatten(void* buffer, size_t size, int fds[], size_t count)
		const {
		if (count != 0) {
		LOGE("flatten: nonzero fd count: %d", count);
		return BAD_VALUE;
		}

		// Write the cache header
		if (size < sizeof(Header)) {
		LOGE("flatten: not enough room for cache header");
		return BAD_VALUE;
		}
		Header* header = reinterpret_cast<Header*>(buffer);
		header->mMagicNumber = blobCacheMagic;
		header->mBlobCacheVersion = blobCacheVersion;
		header->mDeviceVersion = blobCacheDeviceVersion;
		header->mNumEntries = mCacheEntries.size();

		// Write cache entries
		uint8_t* byteBuffer = reinterpret_cast<uint8_t*>(buffer);
		off_t byteOffset = align4(sizeof(Header));
		for (size_t i = 0; i < mCacheEntries.size(); i++) {
		const CacheEntry& e(mCacheEntries[i]);
		sp<Blob> keyBlob = e.getKey();
		sp<Blob> valueBlob = e.getValue();
		size_t keySize = keyBlob->getSize();
		size_t valueSize = valueBlob->getSize();

		size_t entrySize = sizeof(EntryHeader) + keySize + valueSize;
		if (byteOffset + entrySize > size) {
		LOGE("flatten: not enough room for cache entries");
		return BAD_VALUE;
		}

		EntryHeader* eheader = reinterpret_cast<EntryHeader*>(
		&byteBuffer[byteOffset]);
		eheader->mKeySize = keySize;
		eheader->mValueSize = valueSize;

		memcpy(eheader->mData, keyBlob->getData(), keySize);
		memcpy(eheader->mData + keySize, valueBlob->getData(), valueSize);

		byteOffset += align4(entrySize);
		}

		return OK;
		}

		status_t BlobCache::unflatten(void const* buffer, size_t size, int fds[],
		size_t count) {
		// All errors should result in the BlobCache being in an empty state.
		mCacheEntries.clear();

		if (count != 0) {
		LOGE("unflatten: nonzero fd count: %d", count);
		return BAD_VALUE;
		}

		// Read the cache header
		if (size < sizeof(Header)) {
		LOGE("unflatten: not enough room for cache header");
		return BAD_VALUE;
		}
		const Header* header = reinterpret_cast<const Header*>(buffer);
		if (header->mMagicNumber != blobCacheMagic) {
		LOGE("unflatten: bad magic number: %d", header->mMagicNumber);
		return BAD_VALUE;
		}
		if (header->mBlobCacheVersion != blobCacheVersion \|\|
		header->mDeviceVersion != blobCacheDeviceVersion) {
		// We treat version mismatches as an empty cache.
		return OK;
		}

		// Read cache entries
		const uint8_t* byteBuffer = reinterpret_cast<const uint8_t*>(buffer);
		off_t byteOffset = align4(sizeof(Header));
		size_t numEntries = header->mNumEntries;
		for (size_t i = 0; i < numEntries; i++) {
		if (byteOffset + sizeof(EntryHeader) > size) {
		mCacheEntries.clear();
		LOGE("unflatten: not enough room for cache entry headers");
		return BAD_VALUE;
		}

		const EntryHeader* eheader = reinterpret_cast<const EntryHeader*>(
		&byteBuffer[byteOffset]);
		size_t keySize = eheader->mKeySize;
		size_t valueSize = eheader->mValueSize;
		size_t entrySize = sizeof(EntryHeader) + keySize + valueSize;

		if (byteOffset + entrySize > size) {
		mCacheEntries.clear();
		LOGE("unflatten: not enough room for cache entry headers");
		return BAD_VALUE;
		}

		const uint8_t* data = eheader->mData;
		set(data, keySize, data + keySize, valueSize);

		byteOffset += align4(entrySize);
		}

		return OK;
		}

		long int BlobCache::blob_random() {
		#ifdef _WIN32
		return rand();
		@@ -177,7 +313,7 @@ BlobCache::Blob::Blob(const void* data, size_t size, bool copyData):
		mData(copyData ? malloc(size) : data),
		mSize(size),
		mOwnsData(copyData) {
		if (copyData) {
		if (data != NULL && copyData) {
		memcpy(const_cast<void*>(mData), data, size);
		}
		}

libs/utils/tests/BlobCache_test.cpp

+164 −0

Original line number	Diff line number	Diff line
		@@ -14,9 +14,13 @@
		** limitations under the License.
		*/

		#include <fcntl.h>
		#include <stdio.h>

		#include <gtest/gtest.h>

		#include <utils/BlobCache.h>
		#include <utils/Errors.h>

		namespace android {

		@@ -254,4 +258,164 @@ TEST_F(BlobCacheTest, ExceedingTotalLimitHalvesCacheSize) {
		ASSERT_EQ(maxEntries/2 + 1, numCached);
		}

		class BlobCacheFlattenTest : public BlobCacheTest {
		protected:
		virtual void SetUp() {
		BlobCacheTest::SetUp();
		mBC2 = new BlobCache(MAX_KEY_SIZE, MAX_VALUE_SIZE, MAX_TOTAL_SIZE);
		}

		virtual void TearDown() {
		mBC2.clear();
		BlobCacheTest::TearDown();
		}

		void roundTrip() {
		size_t size = mBC->getFlattenedSize();
		uint8_t* flat = new uint8_t[size];
		ASSERT_EQ(OK, mBC->flatten(flat, size, NULL, 0));
		ASSERT_EQ(OK, mBC2->unflatten(flat, size, NULL, 0));
		delete[] flat;
		}

		sp<BlobCache> mBC2;
		};

		TEST_F(BlobCacheFlattenTest, FlattenOneValue) {
		char buf[4] = { 0xee, 0xee, 0xee, 0xee };
		mBC->set("abcd", 4, "efgh", 4);
		roundTrip();
		ASSERT_EQ(size_t(4), mBC2->get("abcd", 4, buf, 4));
		ASSERT_EQ('e', buf[0]);
		ASSERT_EQ('f', buf[1]);
		ASSERT_EQ('g', buf[2]);
		ASSERT_EQ('h', buf[3]);
		}

		TEST_F(BlobCacheFlattenTest, FlattenFullCache) {
		// Fill up the entire cache with 1 char key/value pairs.
		const int maxEntries = MAX_TOTAL_SIZE / 2;
		for (int i = 0; i < maxEntries; i++) {
		uint8_t k = i;
		mBC->set(&k, 1, &k, 1);
		}

		roundTrip();

		// Verify the deserialized cache
		for (int i = 0; i < maxEntries; i++) {
		uint8_t k = i;
		uint8_t v = 0xee;
		ASSERT_EQ(size_t(1), mBC2->get(&k, 1, &v, 1));
		ASSERT_EQ(k, v);
		}
		}

		TEST_F(BlobCacheFlattenTest, FlattenDoesntChangeCache) {
		// Fill up the entire cache with 1 char key/value pairs.
		const int maxEntries = MAX_TOTAL_SIZE / 2;
		for (int i = 0; i < maxEntries; i++) {
		uint8_t k = i;
		mBC->set(&k, 1, &k, 1);
		}

		size_t size = mBC->getFlattenedSize();
		uint8_t* flat = new uint8_t[size];
		ASSERT_EQ(OK, mBC->flatten(flat, size, NULL, 0));
		delete[] flat;

		// Verify the cache that we just serialized
		for (int i = 0; i < maxEntries; i++) {
		uint8_t k = i;
		uint8_t v = 0xee;
		ASSERT_EQ(size_t(1), mBC->get(&k, 1, &v, 1));
		ASSERT_EQ(k, v);
		}
		}

		TEST_F(BlobCacheFlattenTest, FlattenCatchesBufferTooSmall) {
		// Fill up the entire cache with 1 char key/value pairs.
		const int maxEntries = MAX_TOTAL_SIZE / 2;
		for (int i = 0; i < maxEntries; i++) {
		uint8_t k = i;
		mBC->set(&k, 1, &k, 1);
		}

		size_t size = mBC->getFlattenedSize() - 1;
		uint8_t* flat = new uint8_t[size];
		ASSERT_EQ(BAD_VALUE, mBC->flatten(flat, size, NULL, 0));
		delete[] flat;
		}

		TEST_F(BlobCacheFlattenTest, UnflattenCatchesBadMagic) {
		char buf[4] = { 0xee, 0xee, 0xee, 0xee };
		mBC->set("abcd", 4, "efgh", 4);

		size_t size = mBC->getFlattenedSize();
		uint8_t* flat = new uint8_t[size];
		ASSERT_EQ(OK, mBC->flatten(flat, size, NULL, 0));
		flat[1] = ~flat[1];

		// Bad magic should cause an error.
		ASSERT_EQ(BAD_VALUE, mBC2->unflatten(flat, size, NULL, 0));
		delete[] flat;

		// The error should cause the unflatten to result in an empty cache
		ASSERT_EQ(size_t(0), mBC2->get("abcd", 4, buf, 4));
		}

		TEST_F(BlobCacheFlattenTest, UnflattenCatchesBadBlobCacheVersion) {
		char buf[4] = { 0xee, 0xee, 0xee, 0xee };
		mBC->set("abcd", 4, "efgh", 4);

		size_t size = mBC->getFlattenedSize();
		uint8_t* flat = new uint8_t[size];
		ASSERT_EQ(OK, mBC->flatten(flat, size, NULL, 0));
		flat[5] = ~flat[5];

		// Version mismatches shouldn't cause errors, but should not use the
		// serialized entries
		ASSERT_EQ(OK, mBC2->unflatten(flat, size, NULL, 0));
		delete[] flat;

		// The version mismatch should cause the unflatten to result in an empty
		// cache
		ASSERT_EQ(size_t(0), mBC2->get("abcd", 4, buf, 4));
		}

		TEST_F(BlobCacheFlattenTest, UnflattenCatchesBadBlobCacheDeviceVersion) {
		char buf[4] = { 0xee, 0xee, 0xee, 0xee };
		mBC->set("abcd", 4, "efgh", 4);

		size_t size = mBC->getFlattenedSize();
		uint8_t* flat = new uint8_t[size];
		ASSERT_EQ(OK, mBC->flatten(flat, size, NULL, 0));
		flat[10] = ~flat[10];

		// Version mismatches shouldn't cause errors, but should not use the
		// serialized entries
		ASSERT_EQ(OK, mBC2->unflatten(flat, size, NULL, 0));
		delete[] flat;

		// The version mismatch should cause the unflatten to result in an empty
		// cache
		ASSERT_EQ(size_t(0), mBC2->get("abcd", 4, buf, 4));
		}

		TEST_F(BlobCacheFlattenTest, UnflattenCatchesBufferTooSmall) {
		char buf[4] = { 0xee, 0xee, 0xee, 0xee };
		mBC->set("abcd", 4, "efgh", 4);

		size_t size = mBC->getFlattenedSize();
		uint8_t* flat = new uint8_t[size];
		ASSERT_EQ(OK, mBC->flatten(flat, size, NULL, 0));

		// A buffer truncation shouldt cause an error
		ASSERT_EQ(BAD_VALUE, mBC2->unflatten(flat, size-1, NULL, 0));
		delete[] flat;

		// The error should cause the unflatten to result in an empty cache
		ASSERT_EQ(size_t(0), mBC2->get("abcd", 4, buf, 4));
		}

		} // namespace android