Support streaming of compressed assets > 1 megabyte (a45a8008) · Commits · e / os / platform_frameworks_native

include/utils/Asset.h

+2 −9

Original line number	Diff line number	Diff line
		@@ -61,15 +61,6 @@ public:
		ACCESS_BUFFER,
		} AccessMode;

		enum {
		/* data larger than this does not get uncompressed into a buffer */
		#ifdef HAVE_ANDROID_OS
		UNCOMPRESS_DATA_MAX = 1 * 1024 * 1024
		#else
		UNCOMPRESS_DATA_MAX = 2 * 1024 * 1024
		#endif
		};

		/*
		* Read data from the current offset. Returns the actual number of
		* bytes read, 0 on EOF, or -1 on error.
		@@ -317,6 +308,8 @@ private:
		FileMap* mMap; // for memory-mapped input
		int mFd; // for file input

		class StreamingZipInflater* mZipInflater; // for streaming large compressed assets

		unsigned char* mBuf; // for getBuffer()
		};

include/utils/StreamingZipInflater.h

0 → 100644

+82 −0

Original line number	Diff line number	Diff line
		/*
		* Copyright (C) 2010 The Android Open Source Project
		*
		* Licensed under the Apache License, Version 2.0 (the "License");
		* you may not use this file except in compliance with the License.
		* You may obtain a copy of the License at
		*
		* http://www.apache.org/licenses/LICENSE-2.0
		*
		* Unless required by applicable law or agreed to in writing, software
		* distributed under the License is distributed on an "AS IS" BASIS,
		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
		* See the License for the specific language governing permissions and
		* limitations under the License.
		*/

		#ifndef __LIBS_STREAMINGZIPINFLATER_H
		#define __LIBS_STREAMINGZIPINFLATER_H

		#include <unistd.h>
		#include <inttypes.h>
		#include <zlib.h>

		namespace android {

		class StreamingZipInflater {
		public:
		static const size_t INPUT_CHUNK_SIZE = 64 * 1024;
		static const size_t OUTPUT_CHUNK_SIZE = 64 * 1024;

		// Flavor that pages in the compressed data from a fd
		StreamingZipInflater(int fd, off_t compDataStart, size_t uncompSize, size_t compSize);

		// Flavor that gets the compressed data from an in-memory buffer
		StreamingZipInflater(class FileMap* dataMap, size_t uncompSize);

		~StreamingZipInflater();

		// read 'count' bytes of uncompressed data from the current position. outBuf may
		// be NULL, in which case the data is consumed and discarded.
		ssize_t read(void* outBuf, size_t count);

		// seeking backwards requires uncompressing fom the beginning, so is very
		// expensive. seeking forwards only requires uncompressing from the current
		// position to the destination.
		off_t seekAbsolute(off_t absoluteInputPosition);

		private:
		void initInflateState();
		int readNextChunk();

		// where to find the uncompressed data
		int mFd;
		off_t mInFileStart; // where the compressed data lives in the file
		class FileMap* mDataMap;

		z_stream mInflateState;
		bool mStreamNeedsInit;

		// output invariants for this asset
		uint8_t* mOutBuf; // output buf for decompressed bytes
		size_t mOutBufSize; // allocated size of mOutBuf
		size_t mOutTotalSize; // total uncompressed size of the blob

		// current output state bookkeeping
		off_t mOutCurPosition; // current position in total offset
		size_t mOutLastDecoded; // last decoded byte + 1 in mOutbuf
		size_t mOutDeliverable; // next undelivered byte of decoded output in mOutBuf

		// input invariants
		uint8_t* mInBuf;
		size_t mInBufSize; // allocated size of mInBuf;
		size_t mInTotalSize; // total size of compressed data for this blob

		// input state bookkeeping
		size_t mInNextChunkOffset; // offset from start of blob at which the next input chunk lies
		// the z_stream contains state about input block consumption
		};

		}

		#endif

libs/utils/Android.mk

+2 −1

Original line number	Diff line number	Diff line
		@@ -33,6 +33,7 @@ commonSources:= \
		SharedBuffer.cpp \
		Static.cpp \
		StopWatch.cpp \
		StreamingZipInflater.cpp \
		String8.cpp \
		String16.cpp \
		StringArray.cpp \

libs/utils/Asset.cpp

+45 −29

Original line number	Diff line number	Diff line
		@@ -24,6 +24,7 @@
		#include <utils/Asset.h>
		#include <utils/Atomic.h>
		#include <utils/FileMap.h>
		#include <utils/StreamingZipInflater.h>
		#include <utils/ZipUtils.h>
		#include <utils/ZipFileRO.h>
		#include <utils/Log.h>
		@@ -659,7 +660,7 @@ const void* _FileAsset::ensureAlignment(FileMap* map)
		*/
		_CompressedAsset::_CompressedAsset(void)
		: mStart(0), mCompressedLen(0), mUncompressedLen(0), mOffset(0),
		mMap(NULL), mFd(-1), mBuf(NULL)
		mMap(NULL), mFd(-1), mZipInflater(NULL), mBuf(NULL)
		{
		}

		@@ -698,6 +699,10 @@ status_t _CompressedAsset::openChunk(int fd, off_t offset,
		mFd = fd;
		assert(mBuf == NULL);

		if (uncompressedLen > StreamingZipInflater::OUTPUT_CHUNK_SIZE) {
		mZipInflater = new StreamingZipInflater(mFd, offset, uncompressedLen, compressedLen);
		}

		return NO_ERROR;
		}

		@@ -724,6 +729,9 @@ status_t _CompressedAsset::openChunk(FileMap* dataMap, int compressionMethod,
		mUncompressedLen = uncompressedLen;
		assert(mOffset == 0);

		if (uncompressedLen > StreamingZipInflater::OUTPUT_CHUNK_SIZE) {
		mZipInflater = new StreamingZipInflater(dataMap, uncompressedLen);
		}
		return NO_ERROR;
		}

		@@ -739,8 +747,10 @@ ssize_t _CompressedAsset::read(void* buf, size_t count)

		assert(mOffset >= 0 && mOffset <= mUncompressedLen);

		// TODO: if mAccessMode == ACCESS_STREAMING, use zlib more cleverly

		/* If we're relying on a streaming inflater, go through that */
		if (mZipInflater) {
		actual = mZipInflater->read(buf, count);
		} else {
		if (mBuf == NULL) {
		if (getBuffer(false) == NULL)
		return -1;
		@@ -759,6 +769,7 @@ ssize_t _CompressedAsset::read(void* buf, size_t count)
		//printf("comp buf read\n");
		memcpy(buf, (char*)mBuf + mOffset, count);
		actual = count;
		}

		mOffset += actual;
		return actual;
		@@ -780,6 +791,9 @@ off_t _CompressedAsset::seek(off_t offset, int whence)
		if (newPosn == (off_t) -1)
		return newPosn;

		if (mZipInflater) {
		mZipInflater->seekAbsolute(newPosn);
		}
		mOffset = newPosn;
		return mOffset;
		}
		@@ -793,10 +807,12 @@ void _CompressedAsset::close(void)
		mMap->release();
		mMap = NULL;
		}
		if (mBuf != NULL) {

		delete[] mBuf;
		mBuf = NULL;
		}

		delete mZipInflater;
		mZipInflater = NULL;

		if (mFd > 0) {
		::close(mFd);
		@@ -817,12 +833,6 @@ const void* _CompressedAsset::getBuffer(bool wordAligned)
		if (mBuf != NULL)
		return mBuf;

		if (mUncompressedLen > UNCOMPRESS_DATA_MAX) {
		LOGD("Data exceeds UNCOMPRESS_DATA_MAX (%ld vs %d)\n",
		(long) mUncompressedLen, UNCOMPRESS_DATA_MAX);
		goto bail;
		}

		/*
		* Allocate a buffer and read the file into it.
		*/
		@@ -853,7 +863,13 @@ const void* _CompressedAsset::getBuffer(bool wordAligned)
		goto bail;
		}

		/* success! */
		/*
		* Success - now that we have the full asset in RAM we
		* no longer need the streaming inflater
		*/
		delete mZipInflater;
		mZipInflater = NULL;

		mBuf = buf;
		buf = NULL;

libs/utils/StreamingZipInflater.cpp

0 → 100644

+224 −0

Original line number	Diff line number	Diff line
		/*
		* Copyright (C) 2010 The Android Open Source Project
		*
		* Licensed under the Apache License, Version 2.0 (the "License");
		* you may not use this file except in compliance with the License.
		* You may obtain a copy of the License at
		*
		* http://www.apache.org/licenses/LICENSE-2.0
		*
		* Unless required by applicable law or agreed to in writing, software
		* distributed under the License is distributed on an "AS IS" BASIS,
		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
		* See the License for the specific language governing permissions and
		* limitations under the License.
		*/

		#define LOG_TAG "szipinf"
		#include <utils/Log.h>

		#include <utils/FileMap.h>
		#include <utils/StreamingZipInflater.h>
		#include <string.h>
		#include <assert.h>

		static inline size_t min(size_t a, size_t b) { return (a < b) ? a : b; }

		using namespace android;

		/*
		* Streaming access to compressed asset data in an open fd
		*/
		StreamingZipInflater::StreamingZipInflater(int fd, off_t compDataStart,
		size_t uncompSize, size_t compSize) {
		mFd = fd;
		mDataMap = NULL;
		mInFileStart = compDataStart;
		mOutTotalSize = uncompSize;
		mInTotalSize = compSize;

		mInBufSize = StreamingZipInflater::INPUT_CHUNK_SIZE;
		mInBuf = new uint8_t[mInBufSize];

		mOutBufSize = StreamingZipInflater::OUTPUT_CHUNK_SIZE;
		mOutBuf = new uint8_t[mOutBufSize];

		initInflateState();
		}

		/*
		* Streaming access to compressed data held in an mmapped region of memory
		*/
		StreamingZipInflater::StreamingZipInflater(FileMap* dataMap, size_t uncompSize) {
		mFd = -1;
		mDataMap = dataMap;
		mOutTotalSize = uncompSize;
		mInTotalSize = dataMap->getDataLength();

		mInBuf = (uint8_t*) dataMap->getDataPtr();
		mInBufSize = mInTotalSize;

		mOutBufSize = StreamingZipInflater::OUTPUT_CHUNK_SIZE;
		mOutBuf = new uint8_t[mOutBufSize];

		initInflateState();
		}

		StreamingZipInflater::~StreamingZipInflater() {
		// tear down the in-flight zip state just in case
		::inflateEnd(&mInflateState);

		if (mDataMap == NULL) {
		delete [] mInBuf;
		}
		delete [] mOutBuf;
		}

		void StreamingZipInflater::initInflateState() {
		LOGD("Initializing inflate state");

		memset(&mInflateState, 0, sizeof(mInflateState));
		mInflateState.zalloc = Z_NULL;
		mInflateState.zfree = Z_NULL;
		mInflateState.opaque = Z_NULL;
		mInflateState.next_in = (Bytef*)mInBuf;
		mInflateState.next_out = (Bytef*) mOutBuf;
		mInflateState.avail_out = mOutBufSize;
		mInflateState.data_type = Z_UNKNOWN;

		mOutLastDecoded = mOutDeliverable = mOutCurPosition = 0;
		mInNextChunkOffset = 0;
		mStreamNeedsInit = true;

		if (mDataMap == NULL) {
		::lseek(mFd, mInFileStart, SEEK_SET);
		mInflateState.avail_in = 0; // set when a chunk is read in
		} else {
		mInflateState.avail_in = mInBufSize;
		}
		}

		/*
		* Basic approach:
		*
		* 1. If we have undelivered uncompressed data, send it. At this point
		* either we've satisfied the request, or we've exhausted the available
		* output data in mOutBuf.
		*
		* 2. While we haven't sent enough data to satisfy the request:
		* 0. if the request is for more data than exists, bail.
		* a. if there is no input data to decode, read some into the input buffer
		* and readjust the z_stream input pointers
		* b. point the output to the start of the output buffer and decode what we can
		* c. deliver whatever output data we can
		*/
		ssize_t StreamingZipInflater::read(void* outBuf, size_t count) {
		uint8_t* dest = (uint8_t*) outBuf;
		size_t bytesRead = 0;
		size_t toRead = min(count, size_t(mOutTotalSize - mOutCurPosition));
		while (toRead > 0) {
		// First, write from whatever we already have decoded and ready to go
		size_t deliverable = min(toRead, mOutLastDecoded - mOutDeliverable);
		if (deliverable > 0) {
		if (outBuf != NULL) memcpy(dest, mOutBuf + mOutDeliverable, deliverable);
		mOutDeliverable += deliverable;
		mOutCurPosition += deliverable;
		dest += deliverable;
		bytesRead += deliverable;
		toRead -= deliverable;
		}

		// need more data? time to decode some.
		if (toRead > 0) {
		// if we don't have any data to decode, read some in. If we're working
		// from mmapped data this won't happen, because the clipping to total size
		// will prevent reading off the end of the mapped input chunk.
		if (mInflateState.avail_in == 0) {
		int err = readNextChunk();
		if (err < 0) {
		LOGE("Unable to access asset data: %d", err);
		if (!mStreamNeedsInit) {
		::inflateEnd(&mInflateState);
		initInflateState();
		}
		return -1;
		}
		}
		// we know we've drained whatever is in the out buffer now, so just
		// start from scratch there, reading all the input we have at present.
		mInflateState.next_out = (Bytef*) mOutBuf;
		mInflateState.avail_out = mOutBufSize;

		/*
		LOGD("Inflating to outbuf: avail_in=%u avail_out=%u next_in=%p next_out=%p",
		mInflateState.avail_in, mInflateState.avail_out,
		mInflateState.next_in, mInflateState.next_out);
		*/
		int result = Z_OK;
		if (mStreamNeedsInit) {
		LOGI("Initializing zlib to inflate");
		result = inflateInit2(&mInflateState, -MAX_WBITS);
		mStreamNeedsInit = false;
		}
		if (result == Z_OK) result = ::inflate(&mInflateState, Z_SYNC_FLUSH);
		if (result < 0) {
		// Whoops, inflation failed
		LOGE("Error inflating asset: %d", result);
		::inflateEnd(&mInflateState);
		initInflateState();
		return -1;
		} else {
		if (result == Z_STREAM_END) {
		// we know we have to have reached the target size here and will
		// not try to read any further, so just wind things up.
		::inflateEnd(&mInflateState);
		}

		// Note how much data we got, and off we go
		mOutDeliverable = 0;
		mOutLastDecoded = mOutBufSize - mInflateState.avail_out;
		}
		}
		}
		return bytesRead;
		}

		int StreamingZipInflater::readNextChunk() {
		assert(mDataMap == NULL);

		if (mInNextChunkOffset < mInTotalSize) {
		size_t toRead = min(mInBufSize, mInTotalSize - mInNextChunkOffset);
		if (toRead > 0) {
		ssize_t didRead = ::read(mFd, mInBuf, toRead);
		//LOGD("Reading input chunk, size %08x didread %08x", toRead, didRead);
		if (didRead < 0) {
		// TODO: error
		LOGE("Error reading asset data");
		return didRead;
		} else {
		mInNextChunkOffset += didRead;
		mInflateState.next_in = (Bytef*) mInBuf;
		mInflateState.avail_in = didRead;
		}
		}
		}
		return 0;
		}

		// seeking backwards requires uncompressing fom the beginning, so is very
		// expensive. seeking forwards only requires uncompressing from the current
		// position to the destination.
		off_t StreamingZipInflater::seekAbsolute(off_t absoluteInputPosition) {
		if (absoluteInputPosition < mOutCurPosition) {
		// rewind and reprocess the data from the beginning
		if (!mStreamNeedsInit) {
		::inflateEnd(&mInflateState);
		}
		initInflateState();
		read(NULL, absoluteInputPosition);
		} else if (absoluteInputPosition > mOutCurPosition) {
		read(NULL, absoluteInputPosition - mOutCurPosition);
		}
		// else if the target position is our current position, do nothing
		return absoluteInputPosition;
		}