Merge "Support parsing of data descriptor" am: fcf2d8b89b (6476fabe) · Commits · e / os / android_system_core

libziparchive/include/ziparchive/zip_archive.h

+4 −0

Original line number	Diff line number	Diff line
		@@ -77,6 +77,10 @@ struct ZipEntry {
		// footer.
		uint32_t uncompressed_length;

		// If the value of uncompressed length and compressed length are stored in
		// the zip64 extended info of the extra field.
		bool zip64_format_size{false};

		// The offset to the start of data for this ZipEntry.
		off64_t offset;

libziparchive/test_ziparchive_large.py

+24 −4

Original line number	Diff line number	Diff line
		@@ -25,14 +25,18 @@ import zipfile
		import time

		class Zip64Test(unittest.TestCase):
		@staticmethod
		def _WriteFile(path, size_in_kib):
		contents = os.path.basename(path)[0] * 1024
		with open(path, 'w') as f:
		for it in range(0, size_in_kib):
		f.write(contents)

		@staticmethod
		def _AddEntriesToZip(output_zip, entries_dict=None):
		for name, size in entries_dict.items():
		contents = name[0] * 1024
		file_path = tempfile.NamedTemporaryFile()
		with open(file_path.name, 'w') as f:
		for it in range(0, size):
		f.write(contents)
		Zip64Test._WriteFile(file_path.name, size)
		output_zip.write(file_path.name, arcname = name)

		def _getEntryNames(self, zip_name):
		@@ -93,6 +97,22 @@ class Zip64Test(unittest.TestCase):
		self._ExtractEntries(zip_path.name)


		def test_forceDataDescriptor(self):
		file_path = tempfile.NamedTemporaryFile(suffix='.txt')
		# TODO create the entry > 4GiB.
		self._WriteFile(file_path.name, 1024)

		zip_path = tempfile.NamedTemporaryFile(suffix='.zip')
		with zipfile.ZipFile(zip_path, 'w', allowZip64=True) as output_zip:
		pass
		# The fd option force writes a data descriptor
		cmd = ['zip', '-fd', zip_path.name, file_path.name]
		proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
		proc.communicate()
		read_names = self._getEntryNames(zip_path.name)
		self.assertEquals([file_path.name[1:]], read_names)
		self._ExtractEntries(zip_path.name)

		if __name__ == '__main__':
		testsuite = unittest.TestLoader().discover(
		os.path.dirname(os.path.realpath(__file__)))

libziparchive/testdata/zip64.zip

0 → 100644

+200 B

File added.

No diff preview for this file type.

View file

libziparchive/zip_archive.cc

+58 −57

Original line number	Diff line number	Diff line
		@@ -57,8 +57,6 @@
		#include "zip_archive_common.h"
		#include "zip_archive_private.h"

		using android::base::get_unaligned;

		// Used to turn on crc checks - verify that the content CRC matches the values
		// specified in the local file header and the central directory.
		static const bool kCrcChecksEnabled = false;
		@@ -221,7 +219,7 @@ static ZipError FindCentralDirectoryInfo(const char* debug_file_name, ZipArchive
		for (; i >= 0; i--) {
		if (scan_buffer[i] == 0x50) {
		uint32_t* sig_addr = reinterpret_cast<uint32_t*>(&scan_buffer[i]);
		if (get_unaligned<uint32_t>(sig_addr) == EocdRecord::kSignature) {
		if (android::base::get_unaligned<uint32_t>(sig_addr) == EocdRecord::kSignature) {
		ALOGV("+++ Found EOCD at buf+%d", i);
		break;
		}
		@@ -360,8 +358,9 @@ static ZipError ParseZip64ExtendedInfoInExtraField(
		// Data Size - 2 bytes
		uint16_t offset = 0;
		while (offset < extraFieldLength - 4) {
		auto headerId = get_unaligned<uint16_t>(extraFieldStart + offset);
		auto dataSize = get_unaligned<uint16_t>(extraFieldStart + offset + 2);
		auto readPtr = const_cast<uint8_t*>(extraFieldStart + offset);
		auto headerId = ConsumeUnaligned<uint16_t>(&readPtr);
		auto dataSize = ConsumeUnaligned<uint16_t>(&readPtr);

		offset += 4;
		if (dataSize > extraFieldLength - offset) {
		@@ -376,55 +375,45 @@ static ZipError ParseZip64ExtendedInfoInExtraField(
		continue;
		}

		uint16_t expectedDataSize = 0;
		// We expect the extended field to include both uncompressed and compressed size.
		if (zip32UncompressedSize == UINT32_MAX \|\| zip32CompressedSize == UINT32_MAX) {
		expectedDataSize += 16;
		std::optional<uint64_t> uncompressedFileSize;
		std::optional<uint64_t> compressedFileSize;
		std::optional<uint64_t> localHeaderOffset;
		if (zip32UncompressedSize == UINT32_MAX) {
		uncompressedFileSize = ConsumeUnaligned<uint64_t>(&readPtr);
		}
		if (zip32CompressedSize == UINT32_MAX) {
		compressedFileSize = ConsumeUnaligned<uint64_t>(&readPtr);
		}
		if (zip32LocalFileHeaderOffset == UINT32_MAX) {
		expectedDataSize += 8;
		localHeaderOffset = ConsumeUnaligned<uint64_t>(&readPtr);
		}

		if (expectedDataSize == 0) {
		// calculate how many bytes we read after the data size field.
		size_t bytesRead = readPtr - (extraFieldStart + offset);
		if (bytesRead == 0) {
		ALOGW("Zip: Data size should not be 0 in zip64 extended field");
		return kInvalidFile;
		}

		if (dataSize != expectedDataSize) {
		if (dataSize != bytesRead) {
		auto localOffsetString = zip32LocalFileHeaderOffset.has_value()
		? std::to_string(zip32LocalFileHeaderOffset.value())
		: "missing";
		ALOGW("Zip: Invalid data size in zip64 extended field, expect %" PRIu16 ", get %" PRIu16
		ALOGW("Zip: Invalid data size in zip64 extended field, expect %zu , get %" PRIu16
		", uncompressed size %" PRIu32 ", compressed size %" PRIu32 ", local header offset %s",
		expectedDataSize, dataSize, zip32UncompressedSize, zip32CompressedSize,
		bytesRead, dataSize, zip32UncompressedSize, zip32CompressedSize,
		localOffsetString.c_str());
		return kInvalidFile;
		}

		std::optional<uint64_t> uncompressedFileSize;
		std::optional<uint64_t> compressedFileSize;
		std::optional<uint64_t> localHeaderOffset;
		if (zip32UncompressedSize == UINT32_MAX \|\| zip32CompressedSize == UINT32_MAX) {
		uncompressedFileSize = get_unaligned<uint64_t>(extraFieldStart + offset);
		compressedFileSize = get_unaligned<uint64_t>(extraFieldStart + offset + 8);
		offset += 16;

		// TODO(xunchang) Support handling file large than UINT32_MAX. It's theoretically possible
		// for libz to (de)compressing file larger than UINT32_MAX. But we should use our own
		// bytes counter to replace stream.total_out.
		if (uncompressedFileSize.value() >= UINT32_MAX \|\| compressedFileSize.value() >= UINT32_MAX) {
		ALOGW(
		"Zip: File size larger than UINT32_MAX isn't supported yet. uncompressed size %" PRIu64
		", compressed size %" PRIu64,
		uncompressedFileSize.value(), compressedFileSize.value());
		if ((uncompressedFileSize.has_value() && uncompressedFileSize.value() > UINT32_MAX) \|\|
		(compressedFileSize.has_value() && compressedFileSize.value() > UINT32_MAX)) {
		ALOGW("Zip: File size larger than UINT32_MAX isn't supported yet");
		return kInvalidFile;
		}
		}

		if (zip32LocalFileHeaderOffset == UINT32_MAX) {
		localHeaderOffset = get_unaligned<uint64_t>(extraFieldStart + offset);
		offset += 8;
		}

		zip64Info->uncompressed_file_size = uncompressedFileSize;
		zip64Info->compressed_file_size = compressedFileSize;
		@@ -625,7 +614,8 @@ void CloseArchive(ZipArchiveHandle archive) {
		}

		static int32_t ValidateDataDescriptor(MappedZipFile& mapped_zip, ZipEntry* entry) {
		uint8_t ddBuf[sizeof(DataDescriptor) + sizeof(DataDescriptor::kOptSignature)];
		// Maximum possible size for data descriptor: 2 * 4 + 2 * 8 = 24 bytes
		uint8_t ddBuf[24];
		off64_t offset = entry->offset;
		if (entry->method != kCompressStored) {
		offset += entry->compressed_length;
		@@ -638,18 +628,26 @@ static int32_t ValidateDataDescriptor(MappedZipFile& mapped_zip, ZipEntry* entry
		}

		const uint32_t ddSignature = (reinterpret_cast<const uint32_t>(ddBuf));
		const uint16_t ddOffset = (ddSignature == DataDescriptor::kOptSignature) ? 4 : 0;
		const DataDescriptor* descriptor = reinterpret_cast<const DataDescriptor*>(ddBuf + ddOffset);
		uint8_t* ddReadPtr = (ddSignature == DataDescriptor::kOptSignature) ? ddBuf + 4 : ddBuf;
		DataDescriptor descriptor{};
		descriptor.crc32 = ConsumeUnaligned<uint32_t>(&ddReadPtr);
		if (entry->zip64_format_size) {
		descriptor.compressed_size = ConsumeUnaligned<uint64_t>(&ddReadPtr);
		descriptor.uncompressed_size = ConsumeUnaligned<uint64_t>(&ddReadPtr);
		} else {
		descriptor.compressed_size = ConsumeUnaligned<uint32_t>(&ddReadPtr);
		descriptor.uncompressed_size = ConsumeUnaligned<uint32_t>(&ddReadPtr);
		}

		// Validate that the values in the data descriptor match those in the central
		// directory.
		if (entry->compressed_length != descriptor->compressed_size \|\|
		entry->uncompressed_length != descriptor->uncompressed_size \|\|
		entry->crc32 != descriptor->crc32) {
		if (entry->compressed_length != descriptor.compressed_size \|\|
		entry->uncompressed_length != descriptor.uncompressed_size \|\|
		entry->crc32 != descriptor.crc32) {
		ALOGW("Zip: size/crc32 mismatch. expected {%" PRIu32 ", %" PRIu32 ", %" PRIx32
		"}, was {%" PRIu32 ", %" PRIu32 ", %" PRIx32 "}",
		"}, was {%" PRIu64 ", %" PRIu64 ", %" PRIx32 "}",
		entry->compressed_length, entry->uncompressed_length, entry->crc32,
		descriptor->compressed_size, descriptor->uncompressed_size, descriptor->crc32);
		descriptor.compressed_size, descriptor.uncompressed_size, descriptor.crc32);
		return kInconsistentInformation;
		}

		@@ -706,18 +704,14 @@ static int32_t FindEntry(const ZipArchive* archive, std::string_view entryName,
		return status;
		}

		if (cdr->uncompressed_size == UINT32_MAX \|\| cdr->compressed_size == UINT32_MAX) {
		CHECK(zip64_info.uncompressed_file_size.has_value());
		CHECK(zip64_info.compressed_file_size.has_value());
		// TODO(xunchang) remove the size limit and support entry length > UINT32_MAX.
		data->uncompressed_length = static_cast<uint32_t>(zip64_info.uncompressed_file_size.value());
		data->compressed_length = static_cast<uint32_t>(zip64_info.compressed_file_size.value());
		}

		if (local_header_offset == UINT32_MAX) {
		CHECK(zip64_info.local_header_offset.has_value());
		local_header_offset = zip64_info.local_header_offset.value();
		}
		data->uncompressed_length =
		static_cast<uint32_t>(zip64_info.uncompressed_file_size.value_or(cdr->uncompressed_size));
		data->compressed_length =
		static_cast<uint32_t>(zip64_info.compressed_file_size.value_or(cdr->compressed_size));
		local_header_offset = zip64_info.local_header_offset.value_or(local_header_offset);
		data->zip64_format_size =
		cdr->uncompressed_size == UINT32_MAX \|\| cdr->compressed_size == UINT32_MAX;
		}

		if (local_header_offset + static_cast<off64_t>(sizeof(LocalFileHeader)) >= cd_offset) {
		@@ -766,6 +760,13 @@ static int32_t FindEntry(const ZipArchive* archive, std::string_view entryName,
		uint64_t lfh_uncompressed_size = lfh->uncompressed_size;
		uint64_t lfh_compressed_size = lfh->compressed_size;
		if (lfh_uncompressed_size == UINT32_MAX \|\| lfh_compressed_size == UINT32_MAX) {
		if (lfh_uncompressed_size != UINT32_MAX \|\| lfh_compressed_size != UINT32_MAX) {
		ALOGW(
		"Zip: The zip64 extended field in the local header MUST include BOTH original and "
		"compressed file size fields.");
		return kInvalidFile;
		}

		const off64_t lfh_extra_field_offset = name_offset + lfh->file_name_length;
		const uint16_t lfh_extra_field_size = lfh->extra_field_length;
		if (lfh_extra_field_offset > cd_offset - lfh_extra_field_size) {

libziparchive/zip_archive_common.h

+14 −5

Original line number	Diff line number	Diff line
		@@ -165,15 +165,24 @@ struct DataDescriptor {

		// CRC-32 checksum of the entry.
		uint32_t crc32;
		// Compressed size of the entry.
		uint32_t compressed_size;
		// Uncompressed size of the entry.
		uint32_t uncompressed_size;

		// For ZIP64 format archives, the compressed and uncompressed sizes are 8
		// bytes each. Also, the ZIP64 format MAY be used regardless of the size
		// of a file. When extracting, if the zip64 extended information extra field
		// is present for the file the compressed and uncompressed sizes will be 8
		// byte values.

		// Compressed size of the entry, the field can be either 4 bytes or 8 bytes
		// in the zip file.
		uint64_t compressed_size;
		// Uncompressed size of the entry, the field can be either 4 bytes or 8 bytes
		// in the zip file.
		uint64_t uncompressed_size;

		private:
		DataDescriptor() = default;
		DISALLOW_COPY_AND_ASSIGN(DataDescriptor);
		} __attribute__((packed));
		};

		// The zip64 end of central directory locator helps to find the zip64 EOCD.
		struct Zip64EocdLocator {