am bae6fe24: Merge "Support streaming of compressed assets > 1 megabyte" into gingerbread (473aedf5) · Commits · e / os / android_frameworks_native-old

include/utils/Asset.h

+2 −9

Original line number	Original line	Diff line number	Diff line
	@@ -61,15 +61,6 @@ public:
	ACCESS_BUFFER,		ACCESS_BUFFER,
	} AccessMode;		} AccessMode;

	enum {
	/* data larger than this does not get uncompressed into a buffer */
	#ifdef HAVE_ANDROID_OS
	UNCOMPRESS_DATA_MAX = 1 * 1024 * 1024
	#else
	UNCOMPRESS_DATA_MAX = 2 * 1024 * 1024
	#endif
	};

	/*		/*
	* Read data from the current offset. Returns the actual number of		* Read data from the current offset. Returns the actual number of
	* bytes read, 0 on EOF, or -1 on error.		* bytes read, 0 on EOF, or -1 on error.
	@@ -317,6 +308,8 @@ private:
	FileMap* mMap; // for memory-mapped input		FileMap* mMap; // for memory-mapped input
	int mFd; // for file input		int mFd; // for file input

			class StreamingZipInflater* mZipInflater; // for streaming large compressed assets

	unsigned char* mBuf; // for getBuffer()		unsigned char* mBuf; // for getBuffer()
	};		};

include/utils/StreamingZipInflater.h

0 → 100644

+82 −0

Original line number	Original line	Diff line number	Diff line
			/*
			* Copyright (C) 2010 The Android Open Source Project
			*
			* Licensed under the Apache License, Version 2.0 (the "License");
			* you may not use this file except in compliance with the License.
			* You may obtain a copy of the License at
			*
			* http://www.apache.org/licenses/LICENSE-2.0
			*
			* Unless required by applicable law or agreed to in writing, software
			* distributed under the License is distributed on an "AS IS" BASIS,
			* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
			* See the License for the specific language governing permissions and
			* limitations under the License.
			*/

			#ifndef __LIBS_STREAMINGZIPINFLATER_H
			#define __LIBS_STREAMINGZIPINFLATER_H

			#include <unistd.h>
			#include <inttypes.h>
			#include <zlib.h>

			namespace android {

			class StreamingZipInflater {
			public:
			static const size_t INPUT_CHUNK_SIZE = 64 * 1024;
			static const size_t OUTPUT_CHUNK_SIZE = 64 * 1024;

			// Flavor that pages in the compressed data from a fd
			StreamingZipInflater(int fd, off_t compDataStart, size_t uncompSize, size_t compSize);

			// Flavor that gets the compressed data from an in-memory buffer
			StreamingZipInflater(class FileMap* dataMap, size_t uncompSize);

			~StreamingZipInflater();

			// read 'count' bytes of uncompressed data from the current position. outBuf may
			// be NULL, in which case the data is consumed and discarded.
			ssize_t read(void* outBuf, size_t count);

			// seeking backwards requires uncompressing fom the beginning, so is very
			// expensive. seeking forwards only requires uncompressing from the current
			// position to the destination.
			off_t seekAbsolute(off_t absoluteInputPosition);

			private:
			void initInflateState();
			int readNextChunk();

			// where to find the uncompressed data
			int mFd;
			off_t mInFileStart; // where the compressed data lives in the file
			class FileMap* mDataMap;

			z_stream mInflateState;
			bool mStreamNeedsInit;

			// output invariants for this asset
			uint8_t* mOutBuf; // output buf for decompressed bytes
			size_t mOutBufSize; // allocated size of mOutBuf
			size_t mOutTotalSize; // total uncompressed size of the blob

			// current output state bookkeeping
			off_t mOutCurPosition; // current position in total offset
			size_t mOutLastDecoded; // last decoded byte + 1 in mOutbuf
			size_t mOutDeliverable; // next undelivered byte of decoded output in mOutBuf

			// input invariants
			uint8_t* mInBuf;
			size_t mInBufSize; // allocated size of mInBuf;
			size_t mInTotalSize; // total size of compressed data for this blob

			// input state bookkeeping
			size_t mInNextChunkOffset; // offset from start of blob at which the next input chunk lies
			// the z_stream contains state about input block consumption
			};

			}

			#endif

libs/utils/Android.mk

+2 −1

Original line number	Original line	Diff line number	Diff line
	@@ -33,6 +33,7 @@ commonSources:= \
	SharedBuffer.cpp \		SharedBuffer.cpp \
	Static.cpp \		Static.cpp \
	StopWatch.cpp \		StopWatch.cpp \
			StreamingZipInflater.cpp \
	String8.cpp \		String8.cpp \
	String16.cpp \		String16.cpp \
	StringArray.cpp \		StringArray.cpp \

libs/utils/Asset.cpp

+45 −29

Original line number	Original line	Diff line number	Diff line
	@@ -24,6 +24,7 @@
	#include <utils/Asset.h>		#include <utils/Asset.h>
	#include <utils/Atomic.h>		#include <utils/Atomic.h>
	#include <utils/FileMap.h>		#include <utils/FileMap.h>
			#include <utils/StreamingZipInflater.h>
	#include <utils/ZipUtils.h>		#include <utils/ZipUtils.h>
	#include <utils/ZipFileRO.h>		#include <utils/ZipFileRO.h>
	#include <utils/Log.h>		#include <utils/Log.h>
	@@ -659,7 +660,7 @@ const void* _FileAsset::ensureAlignment(FileMap* map)
	*/		*/
	_CompressedAsset::_CompressedAsset(void)		_CompressedAsset::_CompressedAsset(void)
	: mStart(0), mCompressedLen(0), mUncompressedLen(0), mOffset(0),		: mStart(0), mCompressedLen(0), mUncompressedLen(0), mOffset(0),
	mMap(NULL), mFd(-1), mBuf(NULL)		mMap(NULL), mFd(-1), mZipInflater(NULL), mBuf(NULL)
	{		{
	}		}

	@@ -698,6 +699,10 @@ status_t _CompressedAsset::openChunk(int fd, off_t offset,
	mFd = fd;		mFd = fd;
	assert(mBuf == NULL);		assert(mBuf == NULL);

			if (uncompressedLen > StreamingZipInflater::OUTPUT_CHUNK_SIZE) {
			mZipInflater = new StreamingZipInflater(mFd, offset, uncompressedLen, compressedLen);
			}

	return NO_ERROR;		return NO_ERROR;
	}		}

	@@ -724,6 +729,9 @@ status_t _CompressedAsset::openChunk(FileMap* dataMap, int compressionMethod,
	mUncompressedLen = uncompressedLen;		mUncompressedLen = uncompressedLen;
	assert(mOffset == 0);		assert(mOffset == 0);

			if (uncompressedLen > StreamingZipInflater::OUTPUT_CHUNK_SIZE) {
			mZipInflater = new StreamingZipInflater(dataMap, uncompressedLen);
			}
	return NO_ERROR;		return NO_ERROR;
	}		}

	@@ -739,8 +747,10 @@ ssize_t _CompressedAsset::read(void* buf, size_t count)

	assert(mOffset >= 0 && mOffset <= mUncompressedLen);		assert(mOffset >= 0 && mOffset <= mUncompressedLen);

	// TODO: if mAccessMode == ACCESS_STREAMING, use zlib more cleverly		/* If we're relying on a streaming inflater, go through that */
			if (mZipInflater) {
			actual = mZipInflater->read(buf, count);
			} else {
	if (mBuf == NULL) {		if (mBuf == NULL) {
	if (getBuffer(false) == NULL)		if (getBuffer(false) == NULL)
	return -1;		return -1;
	@@ -759,6 +769,7 @@ ssize_t _CompressedAsset::read(void* buf, size_t count)
	//printf("comp buf read\n");		//printf("comp buf read\n");
	memcpy(buf, (char*)mBuf + mOffset, count);		memcpy(buf, (char*)mBuf + mOffset, count);
	actual = count;		actual = count;
			}

	mOffset += actual;		mOffset += actual;
	return actual;		return actual;
	@@ -780,6 +791,9 @@ off_t _CompressedAsset::seek(off_t offset, int whence)
	if (newPosn == (off_t) -1)		if (newPosn == (off_t) -1)
	return newPosn;		return newPosn;

			if (mZipInflater) {
			mZipInflater->seekAbsolute(newPosn);
			}
	mOffset = newPosn;		mOffset = newPosn;
	return mOffset;		return mOffset;
	}		}
	@@ -793,10 +807,12 @@ void _CompressedAsset::close(void)
	mMap->release();		mMap->release();
	mMap = NULL;		mMap = NULL;
	}		}
	if (mBuf != NULL) {
	delete[] mBuf;		delete[] mBuf;
	mBuf = NULL;		mBuf = NULL;
	}
			delete mZipInflater;
			mZipInflater = NULL;

	if (mFd > 0) {		if (mFd > 0) {
	::close(mFd);		::close(mFd);
	@@ -817,12 +833,6 @@ const void* _CompressedAsset::getBuffer(bool wordAligned)
	if (mBuf != NULL)		if (mBuf != NULL)
	return mBuf;		return mBuf;

	if (mUncompressedLen > UNCOMPRESS_DATA_MAX) {
	LOGD("Data exceeds UNCOMPRESS_DATA_MAX (%ld vs %d)\n",
	(long) mUncompressedLen, UNCOMPRESS_DATA_MAX);
	goto bail;
	}

	/*		/*
	* Allocate a buffer and read the file into it.		* Allocate a buffer and read the file into it.
	*/		*/
	@@ -853,7 +863,13 @@ const void* _CompressedAsset::getBuffer(bool wordAligned)
	goto bail;		goto bail;
	}		}

	/* success! */		/*
			* Success - now that we have the full asset in RAM we
			* no longer need the streaming inflater
			*/
			delete mZipInflater;
			mZipInflater = NULL;

	mBuf = buf;		mBuf = buf;
	buf = NULL;		buf = NULL;

libs/utils/StreamingZipInflater.cpp

0 → 100644

+224 −0

Original line number	Original line	Diff line number	Diff line
			/*
			* Copyright (C) 2010 The Android Open Source Project
			*
			* Licensed under the Apache License, Version 2.0 (the "License");
			* you may not use this file except in compliance with the License.
			* You may obtain a copy of the License at
			*
			* http://www.apache.org/licenses/LICENSE-2.0
			*
			* Unless required by applicable law or agreed to in writing, software
			* distributed under the License is distributed on an "AS IS" BASIS,
			* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
			* See the License for the specific language governing permissions and
			* limitations under the License.
			*/

			#define LOG_TAG "szipinf"
			#include <utils/Log.h>

			#include <utils/FileMap.h>
			#include <utils/StreamingZipInflater.h>
			#include <string.h>
			#include <assert.h>

			static inline size_t min(size_t a, size_t b) { return (a < b) ? a : b; }

			using namespace android;

			/*
			* Streaming access to compressed asset data in an open fd
			*/
			StreamingZipInflater::StreamingZipInflater(int fd, off_t compDataStart,
			size_t uncompSize, size_t compSize) {
			mFd = fd;
			mDataMap = NULL;
			mInFileStart = compDataStart;
			mOutTotalSize = uncompSize;
			mInTotalSize = compSize;

			mInBufSize = StreamingZipInflater::INPUT_CHUNK_SIZE;
			mInBuf = new uint8_t[mInBufSize];

			mOutBufSize = StreamingZipInflater::OUTPUT_CHUNK_SIZE;
			mOutBuf = new uint8_t[mOutBufSize];

			initInflateState();
			}

			/*
			* Streaming access to compressed data held in an mmapped region of memory
			*/
			StreamingZipInflater::StreamingZipInflater(FileMap* dataMap, size_t uncompSize) {
			mFd = -1;
			mDataMap = dataMap;
			mOutTotalSize = uncompSize;
			mInTotalSize = dataMap->getDataLength();

			mInBuf = (uint8_t*) dataMap->getDataPtr();
			mInBufSize = mInTotalSize;

			mOutBufSize = StreamingZipInflater::OUTPUT_CHUNK_SIZE;
			mOutBuf = new uint8_t[mOutBufSize];

			initInflateState();
			}

			StreamingZipInflater::~StreamingZipInflater() {
			// tear down the in-flight zip state just in case
			::inflateEnd(&mInflateState);

			if (mDataMap == NULL) {
			delete [] mInBuf;
			}
			delete [] mOutBuf;
			}

			void StreamingZipInflater::initInflateState() {
			LOGD("Initializing inflate state");

			memset(&mInflateState, 0, sizeof(mInflateState));
			mInflateState.zalloc = Z_NULL;
			mInflateState.zfree = Z_NULL;
			mInflateState.opaque = Z_NULL;
			mInflateState.next_in = (Bytef*)mInBuf;
			mInflateState.next_out = (Bytef*) mOutBuf;
			mInflateState.avail_out = mOutBufSize;
			mInflateState.data_type = Z_UNKNOWN;

			mOutLastDecoded = mOutDeliverable = mOutCurPosition = 0;
			mInNextChunkOffset = 0;
			mStreamNeedsInit = true;

			if (mDataMap == NULL) {
			::lseek(mFd, mInFileStart, SEEK_SET);
			mInflateState.avail_in = 0; // set when a chunk is read in
			} else {
			mInflateState.avail_in = mInBufSize;
			}
			}

			/*
			* Basic approach:
			*
			* 1. If we have undelivered uncompressed data, send it. At this point
			* either we've satisfied the request, or we've exhausted the available
			* output data in mOutBuf.
			*
			* 2. While we haven't sent enough data to satisfy the request:
			* 0. if the request is for more data than exists, bail.
			* a. if there is no input data to decode, read some into the input buffer
			* and readjust the z_stream input pointers
			* b. point the output to the start of the output buffer and decode what we can
			* c. deliver whatever output data we can
			*/
			ssize_t StreamingZipInflater::read(void* outBuf, size_t count) {
			uint8_t* dest = (uint8_t*) outBuf;
			size_t bytesRead = 0;
			size_t toRead = min(count, size_t(mOutTotalSize - mOutCurPosition));
			while (toRead > 0) {
			// First, write from whatever we already have decoded and ready to go
			size_t deliverable = min(toRead, mOutLastDecoded - mOutDeliverable);
			if (deliverable > 0) {
			if (outBuf != NULL) memcpy(dest, mOutBuf + mOutDeliverable, deliverable);
			mOutDeliverable += deliverable;
			mOutCurPosition += deliverable;
			dest += deliverable;
			bytesRead += deliverable;
			toRead -= deliverable;
			}

			// need more data? time to decode some.
			if (toRead > 0) {
			// if we don't have any data to decode, read some in. If we're working
			// from mmapped data this won't happen, because the clipping to total size
			// will prevent reading off the end of the mapped input chunk.
			if (mInflateState.avail_in == 0) {
			int err = readNextChunk();
			if (err < 0) {
			LOGE("Unable to access asset data: %d", err);
			if (!mStreamNeedsInit) {
			::inflateEnd(&mInflateState);
			initInflateState();
			}
			return -1;
			}
			}
			// we know we've drained whatever is in the out buffer now, so just
			// start from scratch there, reading all the input we have at present.
			mInflateState.next_out = (Bytef*) mOutBuf;
			mInflateState.avail_out = mOutBufSize;

			/*
			LOGD("Inflating to outbuf: avail_in=%u avail_out=%u next_in=%p next_out=%p",
			mInflateState.avail_in, mInflateState.avail_out,
			mInflateState.next_in, mInflateState.next_out);
			*/
			int result = Z_OK;
			if (mStreamNeedsInit) {
			LOGI("Initializing zlib to inflate");
			result = inflateInit2(&mInflateState, -MAX_WBITS);
			mStreamNeedsInit = false;
			}
			if (result == Z_OK) result = ::inflate(&mInflateState, Z_SYNC_FLUSH);
			if (result < 0) {
			// Whoops, inflation failed
			LOGE("Error inflating asset: %d", result);
			::inflateEnd(&mInflateState);
			initInflateState();
			return -1;
			} else {
			if (result == Z_STREAM_END) {
			// we know we have to have reached the target size here and will
			// not try to read any further, so just wind things up.
			::inflateEnd(&mInflateState);
			}

			// Note how much data we got, and off we go
			mOutDeliverable = 0;
			mOutLastDecoded = mOutBufSize - mInflateState.avail_out;
			}
			}
			}
			return bytesRead;
			}

			int StreamingZipInflater::readNextChunk() {
			assert(mDataMap == NULL);

			if (mInNextChunkOffset < mInTotalSize) {
			size_t toRead = min(mInBufSize, mInTotalSize - mInNextChunkOffset);
			if (toRead > 0) {
			ssize_t didRead = ::read(mFd, mInBuf, toRead);
			//LOGD("Reading input chunk, size %08x didread %08x", toRead, didRead);
			if (didRead < 0) {
			// TODO: error
			LOGE("Error reading asset data");
			return didRead;
			} else {
			mInNextChunkOffset += didRead;
			mInflateState.next_in = (Bytef*) mInBuf;
			mInflateState.avail_in = didRead;
			}
			}
			}
			return 0;
			}

			// seeking backwards requires uncompressing fom the beginning, so is very
			// expensive. seeking forwards only requires uncompressing from the current
			// position to the destination.
			off_t StreamingZipInflater::seekAbsolute(off_t absoluteInputPosition) {
			if (absoluteInputPosition < mOutCurPosition) {
			// rewind and reprocess the data from the beginning
			if (!mStreamNeedsInit) {
			::inflateEnd(&mInflateState);
			}
			initInflateState();
			read(NULL, absoluteInputPosition);
			} else if (absoluteInputPosition > mOutCurPosition) {
			read(NULL, absoluteInputPosition - mOutCurPosition);
			}
			// else if the target position is our current position, do nothing
			return absoluteInputPosition;
			}