Loading libziparchive/include/ziparchive/zip_archive.h +74 −23 Original line number Diff line number Diff line Loading @@ -25,6 +25,7 @@ #include <sys/cdefs.h> #include <sys/types.h> #include <functional> #include <string> #include <string_view> Loading @@ -36,10 +37,10 @@ enum { kCompressDeflated = 8, // standard deflate }; /* * Represents information about a zip entry in a zip file. */ struct ZipEntry { // This struct holds the common information of a zip entry other than the // the entry size. The compressed and uncompressed length will be handled // separately in the derived class. struct ZipEntryCommon { // Compression method. One of kCompressStored or kCompressDeflated. // See also `gpbf` for deflate subtypes. uint16_t method; Loading Loading @@ -67,16 +68,6 @@ struct ZipEntry { // Data descriptor footer at the end of the file entry. uint32_t crc32; // Compressed length of this ZipEntry. Might be present // either in the local file header or in the data descriptor // footer. uint32_t compressed_length; // Uncompressed length of this ZipEntry. Might be present // either in the local file header or in the data descriptor // footer. uint32_t uncompressed_length; // If the value of uncompressed length and compressed length are stored in // the zip64 extended info of the extra field. bool zip64_format_size{false}; Loading @@ -97,6 +88,52 @@ struct ZipEntry { bool is_text; }; struct ZipEntry64; // Many users of the library assume the entry size is capped at UNIT32_MAX. So we keep // the interface for the old ZipEntry here; and we could switch them over to the new // ZipEntry64 later. struct ZipEntry : public ZipEntryCommon { // Compressed length of this ZipEntry. The maximum value is UNIT32_MAX. // Might be present either in the local file header or in the data // descriptor footer. uint32_t compressed_length{0}; // Uncompressed length of this ZipEntry. The maximum value is UNIT32_MAX. // Might be present either in the local file header or in the data // descriptor footer. uint32_t uncompressed_length{0}; // Copies the contents of a ZipEntry64 object to a 32 bits ZipEntry. Returns 0 if the // size of the entry fits into uint32_t, returns a negative error code // (kUnsupportedEntrySize) otherwise. static int32_t CopyFromZipEntry64(ZipEntry* dst, const ZipEntry64* src); private: ZipEntry& operator=(const ZipEntryCommon& other) { ZipEntryCommon::operator=(other); return *this; } }; // Represents information about a zip entry in a zip file. struct ZipEntry64 : public ZipEntryCommon { // Compressed length of this ZipEntry. The maximum value is UNIT64_MAX. // Might be present either in the local file header, the zip64 extended field, // or in the data descriptor footer. uint64_t compressed_length{0}; // Uncompressed length of this ZipEntry. The maximum value is UNIT64_MAX. // Might be present either in the local file header, the zip64 extended field, // or in the data descriptor footer. uint64_t uncompressed_length{0}; explicit ZipEntry64() = default; explicit ZipEntry64(const ZipEntry& zip_entry) : ZipEntryCommon(zip_entry) { compressed_length = zip_entry.compressed_length; uncompressed_length = zip_entry.uncompressed_length; } }; struct ZipArchive; typedef ZipArchive* ZipArchiveHandle; Loading Loading @@ -172,7 +209,8 @@ ZipArchiveInfo GetArchiveInfo(ZipArchiveHandle archive); * On non-Windows platforms this method does not modify internal state and * can be called concurrently. */ int32_t FindEntry(const ZipArchiveHandle archive, const std::string_view entryName, ZipEntry* data); int32_t FindEntry(const ZipArchiveHandle archive, const std::string_view entryName, ZipEntry64* data); /* * Start iterating over all entries of a zip file. The order of iteration Loading Loading @@ -206,8 +244,8 @@ int32_t StartIteration(ZipArchiveHandle archive, void** cookie_ptr, * Returns 0 on success, -1 if there are no more elements in this * archive and lower negative values on failure. */ int32_t Next(void* cookie, ZipEntry* data, std::string* name); int32_t Next(void* cookie, ZipEntry* data, std::string_view* name); int32_t Next(void* cookie, ZipEntry64* data, std::string_view* name); int32_t Next(void* cookie, ZipEntry64* data, std::string* name); /* * End iteration over all entries of a zip file and frees the memory allocated Loading @@ -224,7 +262,7 @@ void EndIteration(void* cookie); * * Returns 0 on success and negative values on failure. */ int32_t ExtractEntryToFile(ZipArchiveHandle archive, ZipEntry* entry, int fd); int32_t ExtractEntryToFile(ZipArchiveHandle archive, const ZipEntry64* entry, int fd); /** * Uncompress a given zip entry to the memory region at |begin| and of Loading @@ -234,7 +272,8 @@ int32_t ExtractEntryToFile(ZipArchiveHandle archive, ZipEntry* entry, int fd); * * Returns 0 on success and negative values on failure. */ int32_t ExtractToMemory(ZipArchiveHandle archive, ZipEntry* entry, uint8_t* begin, uint32_t size); int32_t ExtractToMemory(ZipArchiveHandle archive, const ZipEntry64* entry, uint8_t* begin, size_t size); int GetFileDescriptor(const ZipArchiveHandle archive); Loading @@ -246,6 +285,16 @@ off64_t GetFileDescriptorOffset(const ZipArchiveHandle archive); const char* ErrorCodeString(int32_t error_code); // Many users of libziparchive assume the entry size to be 32 bits long. So we keep these // interfaces that use 32 bit ZipEntry to make old code work. TODO(xunchang) Remove the 32 bit // wrapper functions once we switch all users to recognize ZipEntry64. int32_t FindEntry(const ZipArchiveHandle archive, const std::string_view entryName, ZipEntry* data); int32_t Next(void* cookie, ZipEntry* data, std::string* name); int32_t Next(void* cookie, ZipEntry* data, std::string_view* name); int32_t ExtractEntryToFile(ZipArchiveHandle archive, const ZipEntry* entry, int fd); int32_t ExtractToMemory(ZipArchiveHandle archive, const ZipEntry* entry, uint8_t* begin, size_t size); #if !defined(_WIN32) typedef bool (*ProcessZipEntryFunction)(const uint8_t* buf, size_t buf_size, void* cookie); Loading @@ -253,7 +302,9 @@ typedef bool (*ProcessZipEntryFunction)(const uint8_t* buf, size_t buf_size, voi * Stream the uncompressed data through the supplied function, * passing cookie to it each time it gets called. */ int32_t ProcessZipEntryContents(ZipArchiveHandle archive, ZipEntry* entry, int32_t ProcessZipEntryContents(ZipArchiveHandle archive, const ZipEntry* entry, ProcessZipEntryFunction func, void* cookie); int32_t ProcessZipEntryContents(ZipArchiveHandle archive, const ZipEntry64* entry, ProcessZipEntryFunction func, void* cookie); #endif Loading @@ -274,7 +325,7 @@ class Writer { class Reader { public: virtual bool ReadAtOffset(uint8_t* buf, size_t len, uint32_t offset) const = 0; virtual bool ReadAtOffset(uint8_t* buf, size_t len, off64_t offset) const = 0; virtual ~Reader(); protected: Loading @@ -296,6 +347,6 @@ class Reader { * If |crc_out| is not nullptr, it is set to the crc32 checksum of the * uncompressed data. */ int32_t Inflate(const Reader& reader, const uint32_t compressed_length, const uint32_t uncompressed_length, Writer* writer, uint64_t* crc_out); int32_t Inflate(const Reader& reader, const uint64_t compressed_length, const uint64_t uncompressed_length, Writer* writer, uint64_t* crc_out); } // namespace zip_archive libziparchive/test_ziparchive_large.py +31 −3 Original line number Diff line number Diff line Loading @@ -83,7 +83,7 @@ class Zip64Test(unittest.TestCase): self._ExtractEntries(zip_path.name) def test_largeCompressedEntries(self): def test_largeCompressedEntriesSmallerThan4G(self): zip_path = tempfile.NamedTemporaryFile(suffix='.zip') with zipfile.ZipFile(zip_path, 'w', compression=zipfile.ZIP_DEFLATED, allowZip64=True) as output_zip: Loading @@ -99,8 +99,7 @@ class Zip64Test(unittest.TestCase): def test_forceDataDescriptor(self): file_path = tempfile.NamedTemporaryFile(suffix='.txt') # TODO create the entry > 4GiB. self._WriteFile(file_path.name, 1024) self._WriteFile(file_path.name, 5000 * 1024) zip_path = tempfile.NamedTemporaryFile(suffix='.zip') with zipfile.ZipFile(zip_path, 'w', allowZip64=True) as output_zip: Loading @@ -113,6 +112,35 @@ class Zip64Test(unittest.TestCase): self.assertEquals([file_path.name[1:]], read_names) self._ExtractEntries(zip_path.name) def test_largeUncompressedEntriesLargerThan4G(self): zip_path = tempfile.NamedTemporaryFile(suffix='.zip') with zipfile.ZipFile(zip_path, 'w', compression=zipfile.ZIP_STORED, allowZip64=True) as output_zip: # Add entries close to 4GiB in size. Somehow the python library will put the (un)compressed # sizes in the extra field. Test if our ziptool should be able to parse it. entry_dict = {'g.txt': 5000 * 1024, 'h.txt': 6000 * 1024} self._AddEntriesToZip(output_zip, entry_dict) read_names = self._getEntryNames(zip_path.name) self.assertEquals(sorted(entry_dict.keys()), sorted(read_names)) self._ExtractEntries(zip_path.name) def test_largeCompressedEntriesLargerThan4G(self): zip_path = tempfile.NamedTemporaryFile(suffix='.zip') with zipfile.ZipFile(zip_path, 'w', compression=zipfile.ZIP_DEFLATED, allowZip64=True) as output_zip: # Add entries close to 4GiB in size. Somehow the python library will put the (un)compressed # sizes in the extra field. Test if our ziptool should be able to parse it. entry_dict = {'i.txt': 4096 * 1024, 'j.txt': 7000 * 1024} self._AddEntriesToZip(output_zip, entry_dict) read_names = self._getEntryNames(zip_path.name) self.assertEquals(sorted(entry_dict.keys()), sorted(read_names)) self._ExtractEntries(zip_path.name) if __name__ == '__main__': testsuite = unittest.TestLoader().discover( os.path.dirname(os.path.realpath(__file__))) Loading libziparchive/zip_archive.cc +140 −62 File changed.Preview size limit exceeded, changes collapsed. Show changes libziparchive/zip_archive_private.h +2 −1 Original line number Diff line number Diff line Loading @@ -106,7 +106,8 @@ struct ZipArchive { bool InitializeCentralDirectory(off64_t cd_start_offset, size_t cd_size); }; int32_t ExtractToWriter(ZipArchiveHandle handle, ZipEntry* entry, zip_archive::Writer* writer); int32_t ExtractToWriter(ZipArchiveHandle handle, const ZipEntry64* entry, zip_archive::Writer* writer); // Reads the unaligned data of type |T| and auto increment the offset. template <typename T> Loading libziparchive/zip_archive_test.cc +39 −39 Original line number Diff line number Diff line Loading @@ -217,7 +217,7 @@ TEST(ziparchive, Iteration_std_string_view) { void* iteration_cookie; ASSERT_EQ(0, StartIteration(handle, &iteration_cookie)); ZipEntry data; ZipEntry64 data; std::vector<std::string_view> names; std::string_view name; while (Next(iteration_cookie, &data, &name) == 0) names.push_back(name); Loading @@ -232,12 +232,12 @@ TEST(ziparchive, Iteration_std_string_view) { static void AssertIterationNames(void* iteration_cookie, const std::vector<std::string>& expected_names_sorted) { ZipEntry data; ZipEntry64 data; std::vector<std::string> names; std::string name; std::string_view name; for (size_t i = 0; i < expected_names_sorted.size(); ++i) { ASSERT_EQ(0, Next(iteration_cookie, &data, &name)); names.push_back(name); names.push_back(std::string(name)); } // End of iteration. ASSERT_EQ(-1, Next(iteration_cookie, &data, &name)); Loading Loading @@ -325,8 +325,8 @@ TEST(ziparchive, IterationWithBadPrefixAndSuffix) { void* iteration_cookie; ASSERT_EQ(0, StartIteration(handle, &iteration_cookie, "x", "y")); ZipEntry data; std::string name; ZipEntry64 data; std::string_view name; // End of iteration. ASSERT_EQ(-1, Next(iteration_cookie, &data, &name)); Loading @@ -338,7 +338,7 @@ TEST(ziparchive, FindEntry) { ZipArchiveHandle handle; ASSERT_EQ(0, OpenArchiveWrapper(kValidZip, &handle)); ZipEntry data; ZipEntry64 data; ASSERT_EQ(0, FindEntry(handle, "a.txt", &data)); // Known facts about a.txt, from zipinfo -v. Loading @@ -359,7 +359,7 @@ TEST(ziparchive, FindEntry_empty) { ZipArchiveHandle handle; ASSERT_EQ(0, OpenArchiveWrapper(kValidZip, &handle)); ZipEntry data; ZipEntry64 data; ASSERT_EQ(kInvalidEntryName, FindEntry(handle, "", &data)); CloseArchive(handle); Loading @@ -370,7 +370,7 @@ TEST(ziparchive, FindEntry_too_long) { ASSERT_EQ(0, OpenArchiveWrapper(kValidZip, &handle)); std::string very_long_name(65536, 'x'); ZipEntry data; ZipEntry64 data; ASSERT_EQ(kInvalidEntryName, FindEntry(handle, very_long_name, &data)); CloseArchive(handle); Loading @@ -383,8 +383,8 @@ TEST(ziparchive, TestInvalidDeclaredLength) { void* iteration_cookie; ASSERT_EQ(0, StartIteration(handle, &iteration_cookie)); std::string name; ZipEntry data; std::string_view name; ZipEntry64 data; ASSERT_EQ(Next(iteration_cookie, &data, &name), 0); ASSERT_EQ(Next(iteration_cookie, &data, &name), 0); Loading Loading @@ -415,9 +415,9 @@ TEST(ziparchive, OpenArchiveFdRange) { static_cast<off64_t>(leading_garbage.size()))); // An entry that's deflated. ZipEntry data; ZipEntry64 data; ASSERT_EQ(0, FindEntry(handle, "a.txt", &data)); const uint32_t a_size = data.uncompressed_length; const auto a_size = static_cast<size_t>(data.uncompressed_length); ASSERT_EQ(a_size, kATxtContents.size()); auto buffer = std::unique_ptr<uint8_t[]>(new uint8_t[a_size]); ASSERT_EQ(0, ExtractToMemory(handle, &data, buffer.get(), a_size)); Loading @@ -425,7 +425,7 @@ TEST(ziparchive, OpenArchiveFdRange) { // An entry that's stored. ASSERT_EQ(0, FindEntry(handle, "b.txt", &data)); const uint32_t b_size = data.uncompressed_length; const auto b_size = static_cast<size_t>(data.uncompressed_length); ASSERT_EQ(b_size, kBTxtContents.size()); buffer = std::unique_ptr<uint8_t[]>(new uint8_t[b_size]); ASSERT_EQ(0, ExtractToMemory(handle, &data, buffer.get(), b_size)); Loading @@ -439,9 +439,9 @@ TEST(ziparchive, ExtractToMemory) { ASSERT_EQ(0, OpenArchiveWrapper(kValidZip, &handle)); // An entry that's deflated. ZipEntry data; ZipEntry64 data; ASSERT_EQ(0, FindEntry(handle, "a.txt", &data)); const uint32_t a_size = data.uncompressed_length; const auto a_size = static_cast<size_t>(data.uncompressed_length); ASSERT_EQ(a_size, kATxtContents.size()); uint8_t* buffer = new uint8_t[a_size]; ASSERT_EQ(0, ExtractToMemory(handle, &data, buffer, a_size)); Loading @@ -450,7 +450,7 @@ TEST(ziparchive, ExtractToMemory) { // An entry that's stored. ASSERT_EQ(0, FindEntry(handle, "b.txt", &data)); const uint32_t b_size = data.uncompressed_length; const auto b_size = static_cast<size_t>(data.uncompressed_length); ASSERT_EQ(b_size, kBTxtContents.size()); buffer = new uint8_t[b_size]; ASSERT_EQ(0, ExtractToMemory(handle, &data, buffer, b_size)); Loading Loading @@ -503,7 +503,7 @@ TEST(ziparchive, EmptyEntries) { ZipArchiveHandle handle; ASSERT_EQ(0, OpenArchiveFd(tmp_file.fd, "EmptyEntriesTest", &handle, false)); ZipEntry entry; ZipEntry64 entry; ASSERT_EQ(0, FindEntry(handle, "empty.txt", &entry)); ASSERT_EQ(static_cast<uint32_t>(0), entry.uncompressed_length); uint8_t buffer[1]; Loading @@ -526,7 +526,7 @@ TEST(ziparchive, EntryLargerThan32K) { ZipArchiveHandle handle; ASSERT_EQ(0, OpenArchiveFd(tmp_file.fd, "EntryLargerThan32KTest", &handle, false)); ZipEntry entry; ZipEntry64 entry; ASSERT_EQ(0, FindEntry(handle, kAbTxtName, &entry)); ASSERT_EQ(kAbUncompressedSize, entry.uncompressed_length); Loading Loading @@ -583,7 +583,7 @@ TEST(ziparchive, ExtractToFile) { ZipArchiveHandle handle; ASSERT_EQ(0, OpenArchiveWrapper(kValidZip, &handle)); ZipEntry entry; ZipEntry64 entry; ASSERT_EQ(0, FindEntry(handle, "a.txt", &entry)); ASSERT_EQ(0, ExtractEntryToFile(handle, &entry, tmp_file.fd)); Loading @@ -594,9 +594,9 @@ TEST(ziparchive, ExtractToFile) { ASSERT_EQ(0, memcmp(read_buffer, data, data_size)); // Assert that the remainder of the file contains the incompressed data. std::vector<uint8_t> uncompressed_data(entry.uncompressed_length); ASSERT_TRUE( android::base::ReadFully(tmp_file.fd, uncompressed_data.data(), entry.uncompressed_length)); std::vector<uint8_t> uncompressed_data(static_cast<size_t>(entry.uncompressed_length)); ASSERT_TRUE(android::base::ReadFully(tmp_file.fd, uncompressed_data.data(), static_cast<size_t>(entry.uncompressed_length))); ASSERT_EQ(0, memcmp(&uncompressed_data[0], kATxtContents.data(), kATxtContents.size())); // Assert that the total length of the file is sane Loading @@ -620,7 +620,7 @@ TEST(ziparchive, OpenFromMemory) { OpenArchiveFromMemory(file_map->data(), file_map->size(), zip_path.c_str(), &handle)); // Assert one entry can be found and extracted correctly. ZipEntry binary_entry; ZipEntry64 binary_entry; ASSERT_EQ(0, FindEntry(handle, "META-INF/com/google/android/update-binary", &binary_entry)); TemporaryFile tmp_binary; ASSERT_NE(-1, tmp_binary.fd); Loading @@ -635,13 +635,13 @@ static void ZipArchiveStreamTest(ZipArchiveHandle& handle, const std::string& en if (raw) { stream.reset(ZipArchiveStreamEntry::CreateRaw(handle, *entry)); if (entry->method == kCompressStored) { read_data->resize(entry->uncompressed_length); read_data->resize(static_cast<size_t>(entry->uncompressed_length)); } else { read_data->resize(entry->compressed_length); read_data->resize(static_cast<size_t>(entry->compressed_length)); } } else { stream.reset(ZipArchiveStreamEntry::Create(handle, *entry)); read_data->resize(entry->uncompressed_length); read_data->resize(static_cast<size_t>(entry->uncompressed_length)); } uint8_t* read_data_ptr = read_data->data(); ASSERT_TRUE(stream.get() != nullptr); Loading Loading @@ -681,7 +681,7 @@ static void ZipArchiveStreamTestUsingMemory(const std::string& zip_file, std::vector<uint8_t> read_data; ZipArchiveStreamTest(handle, entry_name, false, true, &entry, &read_data); std::vector<uint8_t> cmp_data(entry.uncompressed_length); std::vector<uint8_t> cmp_data(static_cast<size_t>(entry.uncompressed_length)); ASSERT_EQ(entry.uncompressed_length, read_data.size()); ASSERT_EQ( 0, ExtractToMemory(handle, &entry, cmp_data.data(), static_cast<uint32_t>(cmp_data.size()))); Loading Loading @@ -741,8 +741,8 @@ TEST(ziparchive, StreamUncompressedBadCrc) { // FileOutputStream fos = new // FileOutputStream("/tmp/data_descriptor.zip"); // ZipOutputStream zos = new ZipOutputStream(fos); // ZipEntry ze = new ZipEntry("name"); // ze.setMethod(ZipEntry.DEFLATED); // ZipEntry64 ze = new ZipEntry64("name"); // ze.setMethod(ZipEntry64.DEFLATED); // zos.putNextEntry(ze); // zos.write("abdcdefghijk".getBytes()); // zos.closeEntry(); Loading Loading @@ -780,7 +780,7 @@ static void ExtractEntryToMemory(const std::vector<uint8_t>& zip_data, // This function expects a variant of kDataDescriptorZipFile, for look for // an entry whose name is "name" and whose size is 12 (contents = // "abdcdefghijk"). ZipEntry entry; ZipEntry64 entry; ASSERT_EQ(0, FindEntry(handle, "name", &entry)); ASSERT_EQ(static_cast<uint32_t>(12), entry.uncompressed_length); Loading Loading @@ -887,12 +887,12 @@ class VectorReader : public zip_archive::Reader { public: VectorReader(const std::vector<uint8_t>& input) : Reader(), input_(input) {} bool ReadAtOffset(uint8_t* buf, size_t len, uint32_t offset) const { bool ReadAtOffset(uint8_t* buf, size_t len, off64_t offset) const { if ((offset + len) < input_.size()) { return false; } memcpy(buf, &input_[offset], len); memcpy(buf, &input_[static_cast<size_t>(offset)], len); return true; } Loading @@ -919,7 +919,7 @@ class BadReader : public zip_archive::Reader { public: BadReader() : Reader() {} bool ReadAtOffset(uint8_t*, size_t, uint32_t) const { return false; } bool ReadAtOffset(uint8_t*, size_t, off64_t) const { return false; } }; class BadWriter : public zip_archive::Writer { Loading Loading @@ -1222,7 +1222,7 @@ TEST_F(Zip64ParseTest, findEntry) { ZipArchiveHandle handle; ASSERT_EQ( 0, OpenArchiveFromMemory(zip_content_.data(), zip_content_.size(), "debug_zip64", &handle)); ZipEntry entry; ZipEntry64 entry; ASSERT_EQ(0, FindEntry(handle, "a.txt", &entry)); ASSERT_EQ(200, entry.uncompressed_length); ASSERT_EQ(200, entry.compressed_length); Loading @@ -1245,7 +1245,7 @@ TEST_F(Zip64ParseTest, openFileIncorrectDataSizeInLocalExtendedField) { ZipArchiveHandle handle; ASSERT_EQ( 0, OpenArchiveFromMemory(zip_content_.data(), zip_content_.size(), "debug_zip64", &handle)); ZipEntry entry; ZipEntry64 entry; ASSERT_NE(0, FindEntry(handle, "a.txt", &entry)); CloseArchive(handle); Loading @@ -1267,7 +1267,7 @@ TEST_F(Zip64ParseTest, iterates) { ASSERT_EQ(0, StartIteration(handle, &iteration_cookie)); std::set<std::string_view> result; std::string_view name; ZipEntry entry; ZipEntry64 entry; while (Next(iteration_cookie, &entry, &name) == 0) result.emplace(name); ASSERT_EQ(names, result); Loading Loading @@ -1297,7 +1297,7 @@ TEST_F(Zip64ParseTest, extract) { ZipArchiveHandle handle; ASSERT_EQ( 0, OpenArchiveFromMemory(zip_content_.data(), zip_content_.size(), "debug_zip64", &handle)); ZipEntry entry; ZipEntry64 entry; ASSERT_EQ(0, FindEntry(handle, "a.txt", &entry)); VectorWriter writer; Loading @@ -1315,7 +1315,7 @@ TEST_F(Zip64ParseTest, extractWithDataDescriptor) { ZipArchiveHandle handle; ASSERT_EQ( 0, OpenArchiveFromMemory(zip_content_.data(), zip_content_.size(), "debug_zip64", &handle)); ZipEntry entry; ZipEntry64 entry; ASSERT_EQ(0, FindEntry(handle, "b.txt", &entry)); VectorWriter writer; Loading Loading
libziparchive/include/ziparchive/zip_archive.h +74 −23 Original line number Diff line number Diff line Loading @@ -25,6 +25,7 @@ #include <sys/cdefs.h> #include <sys/types.h> #include <functional> #include <string> #include <string_view> Loading @@ -36,10 +37,10 @@ enum { kCompressDeflated = 8, // standard deflate }; /* * Represents information about a zip entry in a zip file. */ struct ZipEntry { // This struct holds the common information of a zip entry other than the // the entry size. The compressed and uncompressed length will be handled // separately in the derived class. struct ZipEntryCommon { // Compression method. One of kCompressStored or kCompressDeflated. // See also `gpbf` for deflate subtypes. uint16_t method; Loading Loading @@ -67,16 +68,6 @@ struct ZipEntry { // Data descriptor footer at the end of the file entry. uint32_t crc32; // Compressed length of this ZipEntry. Might be present // either in the local file header or in the data descriptor // footer. uint32_t compressed_length; // Uncompressed length of this ZipEntry. Might be present // either in the local file header or in the data descriptor // footer. uint32_t uncompressed_length; // If the value of uncompressed length and compressed length are stored in // the zip64 extended info of the extra field. bool zip64_format_size{false}; Loading @@ -97,6 +88,52 @@ struct ZipEntry { bool is_text; }; struct ZipEntry64; // Many users of the library assume the entry size is capped at UNIT32_MAX. So we keep // the interface for the old ZipEntry here; and we could switch them over to the new // ZipEntry64 later. struct ZipEntry : public ZipEntryCommon { // Compressed length of this ZipEntry. The maximum value is UNIT32_MAX. // Might be present either in the local file header or in the data // descriptor footer. uint32_t compressed_length{0}; // Uncompressed length of this ZipEntry. The maximum value is UNIT32_MAX. // Might be present either in the local file header or in the data // descriptor footer. uint32_t uncompressed_length{0}; // Copies the contents of a ZipEntry64 object to a 32 bits ZipEntry. Returns 0 if the // size of the entry fits into uint32_t, returns a negative error code // (kUnsupportedEntrySize) otherwise. static int32_t CopyFromZipEntry64(ZipEntry* dst, const ZipEntry64* src); private: ZipEntry& operator=(const ZipEntryCommon& other) { ZipEntryCommon::operator=(other); return *this; } }; // Represents information about a zip entry in a zip file. struct ZipEntry64 : public ZipEntryCommon { // Compressed length of this ZipEntry. The maximum value is UNIT64_MAX. // Might be present either in the local file header, the zip64 extended field, // or in the data descriptor footer. uint64_t compressed_length{0}; // Uncompressed length of this ZipEntry. The maximum value is UNIT64_MAX. // Might be present either in the local file header, the zip64 extended field, // or in the data descriptor footer. uint64_t uncompressed_length{0}; explicit ZipEntry64() = default; explicit ZipEntry64(const ZipEntry& zip_entry) : ZipEntryCommon(zip_entry) { compressed_length = zip_entry.compressed_length; uncompressed_length = zip_entry.uncompressed_length; } }; struct ZipArchive; typedef ZipArchive* ZipArchiveHandle; Loading Loading @@ -172,7 +209,8 @@ ZipArchiveInfo GetArchiveInfo(ZipArchiveHandle archive); * On non-Windows platforms this method does not modify internal state and * can be called concurrently. */ int32_t FindEntry(const ZipArchiveHandle archive, const std::string_view entryName, ZipEntry* data); int32_t FindEntry(const ZipArchiveHandle archive, const std::string_view entryName, ZipEntry64* data); /* * Start iterating over all entries of a zip file. The order of iteration Loading Loading @@ -206,8 +244,8 @@ int32_t StartIteration(ZipArchiveHandle archive, void** cookie_ptr, * Returns 0 on success, -1 if there are no more elements in this * archive and lower negative values on failure. */ int32_t Next(void* cookie, ZipEntry* data, std::string* name); int32_t Next(void* cookie, ZipEntry* data, std::string_view* name); int32_t Next(void* cookie, ZipEntry64* data, std::string_view* name); int32_t Next(void* cookie, ZipEntry64* data, std::string* name); /* * End iteration over all entries of a zip file and frees the memory allocated Loading @@ -224,7 +262,7 @@ void EndIteration(void* cookie); * * Returns 0 on success and negative values on failure. */ int32_t ExtractEntryToFile(ZipArchiveHandle archive, ZipEntry* entry, int fd); int32_t ExtractEntryToFile(ZipArchiveHandle archive, const ZipEntry64* entry, int fd); /** * Uncompress a given zip entry to the memory region at |begin| and of Loading @@ -234,7 +272,8 @@ int32_t ExtractEntryToFile(ZipArchiveHandle archive, ZipEntry* entry, int fd); * * Returns 0 on success and negative values on failure. */ int32_t ExtractToMemory(ZipArchiveHandle archive, ZipEntry* entry, uint8_t* begin, uint32_t size); int32_t ExtractToMemory(ZipArchiveHandle archive, const ZipEntry64* entry, uint8_t* begin, size_t size); int GetFileDescriptor(const ZipArchiveHandle archive); Loading @@ -246,6 +285,16 @@ off64_t GetFileDescriptorOffset(const ZipArchiveHandle archive); const char* ErrorCodeString(int32_t error_code); // Many users of libziparchive assume the entry size to be 32 bits long. So we keep these // interfaces that use 32 bit ZipEntry to make old code work. TODO(xunchang) Remove the 32 bit // wrapper functions once we switch all users to recognize ZipEntry64. int32_t FindEntry(const ZipArchiveHandle archive, const std::string_view entryName, ZipEntry* data); int32_t Next(void* cookie, ZipEntry* data, std::string* name); int32_t Next(void* cookie, ZipEntry* data, std::string_view* name); int32_t ExtractEntryToFile(ZipArchiveHandle archive, const ZipEntry* entry, int fd); int32_t ExtractToMemory(ZipArchiveHandle archive, const ZipEntry* entry, uint8_t* begin, size_t size); #if !defined(_WIN32) typedef bool (*ProcessZipEntryFunction)(const uint8_t* buf, size_t buf_size, void* cookie); Loading @@ -253,7 +302,9 @@ typedef bool (*ProcessZipEntryFunction)(const uint8_t* buf, size_t buf_size, voi * Stream the uncompressed data through the supplied function, * passing cookie to it each time it gets called. */ int32_t ProcessZipEntryContents(ZipArchiveHandle archive, ZipEntry* entry, int32_t ProcessZipEntryContents(ZipArchiveHandle archive, const ZipEntry* entry, ProcessZipEntryFunction func, void* cookie); int32_t ProcessZipEntryContents(ZipArchiveHandle archive, const ZipEntry64* entry, ProcessZipEntryFunction func, void* cookie); #endif Loading @@ -274,7 +325,7 @@ class Writer { class Reader { public: virtual bool ReadAtOffset(uint8_t* buf, size_t len, uint32_t offset) const = 0; virtual bool ReadAtOffset(uint8_t* buf, size_t len, off64_t offset) const = 0; virtual ~Reader(); protected: Loading @@ -296,6 +347,6 @@ class Reader { * If |crc_out| is not nullptr, it is set to the crc32 checksum of the * uncompressed data. */ int32_t Inflate(const Reader& reader, const uint32_t compressed_length, const uint32_t uncompressed_length, Writer* writer, uint64_t* crc_out); int32_t Inflate(const Reader& reader, const uint64_t compressed_length, const uint64_t uncompressed_length, Writer* writer, uint64_t* crc_out); } // namespace zip_archive
libziparchive/test_ziparchive_large.py +31 −3 Original line number Diff line number Diff line Loading @@ -83,7 +83,7 @@ class Zip64Test(unittest.TestCase): self._ExtractEntries(zip_path.name) def test_largeCompressedEntries(self): def test_largeCompressedEntriesSmallerThan4G(self): zip_path = tempfile.NamedTemporaryFile(suffix='.zip') with zipfile.ZipFile(zip_path, 'w', compression=zipfile.ZIP_DEFLATED, allowZip64=True) as output_zip: Loading @@ -99,8 +99,7 @@ class Zip64Test(unittest.TestCase): def test_forceDataDescriptor(self): file_path = tempfile.NamedTemporaryFile(suffix='.txt') # TODO create the entry > 4GiB. self._WriteFile(file_path.name, 1024) self._WriteFile(file_path.name, 5000 * 1024) zip_path = tempfile.NamedTemporaryFile(suffix='.zip') with zipfile.ZipFile(zip_path, 'w', allowZip64=True) as output_zip: Loading @@ -113,6 +112,35 @@ class Zip64Test(unittest.TestCase): self.assertEquals([file_path.name[1:]], read_names) self._ExtractEntries(zip_path.name) def test_largeUncompressedEntriesLargerThan4G(self): zip_path = tempfile.NamedTemporaryFile(suffix='.zip') with zipfile.ZipFile(zip_path, 'w', compression=zipfile.ZIP_STORED, allowZip64=True) as output_zip: # Add entries close to 4GiB in size. Somehow the python library will put the (un)compressed # sizes in the extra field. Test if our ziptool should be able to parse it. entry_dict = {'g.txt': 5000 * 1024, 'h.txt': 6000 * 1024} self._AddEntriesToZip(output_zip, entry_dict) read_names = self._getEntryNames(zip_path.name) self.assertEquals(sorted(entry_dict.keys()), sorted(read_names)) self._ExtractEntries(zip_path.name) def test_largeCompressedEntriesLargerThan4G(self): zip_path = tempfile.NamedTemporaryFile(suffix='.zip') with zipfile.ZipFile(zip_path, 'w', compression=zipfile.ZIP_DEFLATED, allowZip64=True) as output_zip: # Add entries close to 4GiB in size. Somehow the python library will put the (un)compressed # sizes in the extra field. Test if our ziptool should be able to parse it. entry_dict = {'i.txt': 4096 * 1024, 'j.txt': 7000 * 1024} self._AddEntriesToZip(output_zip, entry_dict) read_names = self._getEntryNames(zip_path.name) self.assertEquals(sorted(entry_dict.keys()), sorted(read_names)) self._ExtractEntries(zip_path.name) if __name__ == '__main__': testsuite = unittest.TestLoader().discover( os.path.dirname(os.path.realpath(__file__))) Loading
libziparchive/zip_archive.cc +140 −62 File changed.Preview size limit exceeded, changes collapsed. Show changes
libziparchive/zip_archive_private.h +2 −1 Original line number Diff line number Diff line Loading @@ -106,7 +106,8 @@ struct ZipArchive { bool InitializeCentralDirectory(off64_t cd_start_offset, size_t cd_size); }; int32_t ExtractToWriter(ZipArchiveHandle handle, ZipEntry* entry, zip_archive::Writer* writer); int32_t ExtractToWriter(ZipArchiveHandle handle, const ZipEntry64* entry, zip_archive::Writer* writer); // Reads the unaligned data of type |T| and auto increment the offset. template <typename T> Loading
libziparchive/zip_archive_test.cc +39 −39 Original line number Diff line number Diff line Loading @@ -217,7 +217,7 @@ TEST(ziparchive, Iteration_std_string_view) { void* iteration_cookie; ASSERT_EQ(0, StartIteration(handle, &iteration_cookie)); ZipEntry data; ZipEntry64 data; std::vector<std::string_view> names; std::string_view name; while (Next(iteration_cookie, &data, &name) == 0) names.push_back(name); Loading @@ -232,12 +232,12 @@ TEST(ziparchive, Iteration_std_string_view) { static void AssertIterationNames(void* iteration_cookie, const std::vector<std::string>& expected_names_sorted) { ZipEntry data; ZipEntry64 data; std::vector<std::string> names; std::string name; std::string_view name; for (size_t i = 0; i < expected_names_sorted.size(); ++i) { ASSERT_EQ(0, Next(iteration_cookie, &data, &name)); names.push_back(name); names.push_back(std::string(name)); } // End of iteration. ASSERT_EQ(-1, Next(iteration_cookie, &data, &name)); Loading Loading @@ -325,8 +325,8 @@ TEST(ziparchive, IterationWithBadPrefixAndSuffix) { void* iteration_cookie; ASSERT_EQ(0, StartIteration(handle, &iteration_cookie, "x", "y")); ZipEntry data; std::string name; ZipEntry64 data; std::string_view name; // End of iteration. ASSERT_EQ(-1, Next(iteration_cookie, &data, &name)); Loading @@ -338,7 +338,7 @@ TEST(ziparchive, FindEntry) { ZipArchiveHandle handle; ASSERT_EQ(0, OpenArchiveWrapper(kValidZip, &handle)); ZipEntry data; ZipEntry64 data; ASSERT_EQ(0, FindEntry(handle, "a.txt", &data)); // Known facts about a.txt, from zipinfo -v. Loading @@ -359,7 +359,7 @@ TEST(ziparchive, FindEntry_empty) { ZipArchiveHandle handle; ASSERT_EQ(0, OpenArchiveWrapper(kValidZip, &handle)); ZipEntry data; ZipEntry64 data; ASSERT_EQ(kInvalidEntryName, FindEntry(handle, "", &data)); CloseArchive(handle); Loading @@ -370,7 +370,7 @@ TEST(ziparchive, FindEntry_too_long) { ASSERT_EQ(0, OpenArchiveWrapper(kValidZip, &handle)); std::string very_long_name(65536, 'x'); ZipEntry data; ZipEntry64 data; ASSERT_EQ(kInvalidEntryName, FindEntry(handle, very_long_name, &data)); CloseArchive(handle); Loading @@ -383,8 +383,8 @@ TEST(ziparchive, TestInvalidDeclaredLength) { void* iteration_cookie; ASSERT_EQ(0, StartIteration(handle, &iteration_cookie)); std::string name; ZipEntry data; std::string_view name; ZipEntry64 data; ASSERT_EQ(Next(iteration_cookie, &data, &name), 0); ASSERT_EQ(Next(iteration_cookie, &data, &name), 0); Loading Loading @@ -415,9 +415,9 @@ TEST(ziparchive, OpenArchiveFdRange) { static_cast<off64_t>(leading_garbage.size()))); // An entry that's deflated. ZipEntry data; ZipEntry64 data; ASSERT_EQ(0, FindEntry(handle, "a.txt", &data)); const uint32_t a_size = data.uncompressed_length; const auto a_size = static_cast<size_t>(data.uncompressed_length); ASSERT_EQ(a_size, kATxtContents.size()); auto buffer = std::unique_ptr<uint8_t[]>(new uint8_t[a_size]); ASSERT_EQ(0, ExtractToMemory(handle, &data, buffer.get(), a_size)); Loading @@ -425,7 +425,7 @@ TEST(ziparchive, OpenArchiveFdRange) { // An entry that's stored. ASSERT_EQ(0, FindEntry(handle, "b.txt", &data)); const uint32_t b_size = data.uncompressed_length; const auto b_size = static_cast<size_t>(data.uncompressed_length); ASSERT_EQ(b_size, kBTxtContents.size()); buffer = std::unique_ptr<uint8_t[]>(new uint8_t[b_size]); ASSERT_EQ(0, ExtractToMemory(handle, &data, buffer.get(), b_size)); Loading @@ -439,9 +439,9 @@ TEST(ziparchive, ExtractToMemory) { ASSERT_EQ(0, OpenArchiveWrapper(kValidZip, &handle)); // An entry that's deflated. ZipEntry data; ZipEntry64 data; ASSERT_EQ(0, FindEntry(handle, "a.txt", &data)); const uint32_t a_size = data.uncompressed_length; const auto a_size = static_cast<size_t>(data.uncompressed_length); ASSERT_EQ(a_size, kATxtContents.size()); uint8_t* buffer = new uint8_t[a_size]; ASSERT_EQ(0, ExtractToMemory(handle, &data, buffer, a_size)); Loading @@ -450,7 +450,7 @@ TEST(ziparchive, ExtractToMemory) { // An entry that's stored. ASSERT_EQ(0, FindEntry(handle, "b.txt", &data)); const uint32_t b_size = data.uncompressed_length; const auto b_size = static_cast<size_t>(data.uncompressed_length); ASSERT_EQ(b_size, kBTxtContents.size()); buffer = new uint8_t[b_size]; ASSERT_EQ(0, ExtractToMemory(handle, &data, buffer, b_size)); Loading Loading @@ -503,7 +503,7 @@ TEST(ziparchive, EmptyEntries) { ZipArchiveHandle handle; ASSERT_EQ(0, OpenArchiveFd(tmp_file.fd, "EmptyEntriesTest", &handle, false)); ZipEntry entry; ZipEntry64 entry; ASSERT_EQ(0, FindEntry(handle, "empty.txt", &entry)); ASSERT_EQ(static_cast<uint32_t>(0), entry.uncompressed_length); uint8_t buffer[1]; Loading @@ -526,7 +526,7 @@ TEST(ziparchive, EntryLargerThan32K) { ZipArchiveHandle handle; ASSERT_EQ(0, OpenArchiveFd(tmp_file.fd, "EntryLargerThan32KTest", &handle, false)); ZipEntry entry; ZipEntry64 entry; ASSERT_EQ(0, FindEntry(handle, kAbTxtName, &entry)); ASSERT_EQ(kAbUncompressedSize, entry.uncompressed_length); Loading Loading @@ -583,7 +583,7 @@ TEST(ziparchive, ExtractToFile) { ZipArchiveHandle handle; ASSERT_EQ(0, OpenArchiveWrapper(kValidZip, &handle)); ZipEntry entry; ZipEntry64 entry; ASSERT_EQ(0, FindEntry(handle, "a.txt", &entry)); ASSERT_EQ(0, ExtractEntryToFile(handle, &entry, tmp_file.fd)); Loading @@ -594,9 +594,9 @@ TEST(ziparchive, ExtractToFile) { ASSERT_EQ(0, memcmp(read_buffer, data, data_size)); // Assert that the remainder of the file contains the incompressed data. std::vector<uint8_t> uncompressed_data(entry.uncompressed_length); ASSERT_TRUE( android::base::ReadFully(tmp_file.fd, uncompressed_data.data(), entry.uncompressed_length)); std::vector<uint8_t> uncompressed_data(static_cast<size_t>(entry.uncompressed_length)); ASSERT_TRUE(android::base::ReadFully(tmp_file.fd, uncompressed_data.data(), static_cast<size_t>(entry.uncompressed_length))); ASSERT_EQ(0, memcmp(&uncompressed_data[0], kATxtContents.data(), kATxtContents.size())); // Assert that the total length of the file is sane Loading @@ -620,7 +620,7 @@ TEST(ziparchive, OpenFromMemory) { OpenArchiveFromMemory(file_map->data(), file_map->size(), zip_path.c_str(), &handle)); // Assert one entry can be found and extracted correctly. ZipEntry binary_entry; ZipEntry64 binary_entry; ASSERT_EQ(0, FindEntry(handle, "META-INF/com/google/android/update-binary", &binary_entry)); TemporaryFile tmp_binary; ASSERT_NE(-1, tmp_binary.fd); Loading @@ -635,13 +635,13 @@ static void ZipArchiveStreamTest(ZipArchiveHandle& handle, const std::string& en if (raw) { stream.reset(ZipArchiveStreamEntry::CreateRaw(handle, *entry)); if (entry->method == kCompressStored) { read_data->resize(entry->uncompressed_length); read_data->resize(static_cast<size_t>(entry->uncompressed_length)); } else { read_data->resize(entry->compressed_length); read_data->resize(static_cast<size_t>(entry->compressed_length)); } } else { stream.reset(ZipArchiveStreamEntry::Create(handle, *entry)); read_data->resize(entry->uncompressed_length); read_data->resize(static_cast<size_t>(entry->uncompressed_length)); } uint8_t* read_data_ptr = read_data->data(); ASSERT_TRUE(stream.get() != nullptr); Loading Loading @@ -681,7 +681,7 @@ static void ZipArchiveStreamTestUsingMemory(const std::string& zip_file, std::vector<uint8_t> read_data; ZipArchiveStreamTest(handle, entry_name, false, true, &entry, &read_data); std::vector<uint8_t> cmp_data(entry.uncompressed_length); std::vector<uint8_t> cmp_data(static_cast<size_t>(entry.uncompressed_length)); ASSERT_EQ(entry.uncompressed_length, read_data.size()); ASSERT_EQ( 0, ExtractToMemory(handle, &entry, cmp_data.data(), static_cast<uint32_t>(cmp_data.size()))); Loading Loading @@ -741,8 +741,8 @@ TEST(ziparchive, StreamUncompressedBadCrc) { // FileOutputStream fos = new // FileOutputStream("/tmp/data_descriptor.zip"); // ZipOutputStream zos = new ZipOutputStream(fos); // ZipEntry ze = new ZipEntry("name"); // ze.setMethod(ZipEntry.DEFLATED); // ZipEntry64 ze = new ZipEntry64("name"); // ze.setMethod(ZipEntry64.DEFLATED); // zos.putNextEntry(ze); // zos.write("abdcdefghijk".getBytes()); // zos.closeEntry(); Loading Loading @@ -780,7 +780,7 @@ static void ExtractEntryToMemory(const std::vector<uint8_t>& zip_data, // This function expects a variant of kDataDescriptorZipFile, for look for // an entry whose name is "name" and whose size is 12 (contents = // "abdcdefghijk"). ZipEntry entry; ZipEntry64 entry; ASSERT_EQ(0, FindEntry(handle, "name", &entry)); ASSERT_EQ(static_cast<uint32_t>(12), entry.uncompressed_length); Loading Loading @@ -887,12 +887,12 @@ class VectorReader : public zip_archive::Reader { public: VectorReader(const std::vector<uint8_t>& input) : Reader(), input_(input) {} bool ReadAtOffset(uint8_t* buf, size_t len, uint32_t offset) const { bool ReadAtOffset(uint8_t* buf, size_t len, off64_t offset) const { if ((offset + len) < input_.size()) { return false; } memcpy(buf, &input_[offset], len); memcpy(buf, &input_[static_cast<size_t>(offset)], len); return true; } Loading @@ -919,7 +919,7 @@ class BadReader : public zip_archive::Reader { public: BadReader() : Reader() {} bool ReadAtOffset(uint8_t*, size_t, uint32_t) const { return false; } bool ReadAtOffset(uint8_t*, size_t, off64_t) const { return false; } }; class BadWriter : public zip_archive::Writer { Loading Loading @@ -1222,7 +1222,7 @@ TEST_F(Zip64ParseTest, findEntry) { ZipArchiveHandle handle; ASSERT_EQ( 0, OpenArchiveFromMemory(zip_content_.data(), zip_content_.size(), "debug_zip64", &handle)); ZipEntry entry; ZipEntry64 entry; ASSERT_EQ(0, FindEntry(handle, "a.txt", &entry)); ASSERT_EQ(200, entry.uncompressed_length); ASSERT_EQ(200, entry.compressed_length); Loading @@ -1245,7 +1245,7 @@ TEST_F(Zip64ParseTest, openFileIncorrectDataSizeInLocalExtendedField) { ZipArchiveHandle handle; ASSERT_EQ( 0, OpenArchiveFromMemory(zip_content_.data(), zip_content_.size(), "debug_zip64", &handle)); ZipEntry entry; ZipEntry64 entry; ASSERT_NE(0, FindEntry(handle, "a.txt", &entry)); CloseArchive(handle); Loading @@ -1267,7 +1267,7 @@ TEST_F(Zip64ParseTest, iterates) { ASSERT_EQ(0, StartIteration(handle, &iteration_cookie)); std::set<std::string_view> result; std::string_view name; ZipEntry entry; ZipEntry64 entry; while (Next(iteration_cookie, &entry, &name) == 0) result.emplace(name); ASSERT_EQ(names, result); Loading Loading @@ -1297,7 +1297,7 @@ TEST_F(Zip64ParseTest, extract) { ZipArchiveHandle handle; ASSERT_EQ( 0, OpenArchiveFromMemory(zip_content_.data(), zip_content_.size(), "debug_zip64", &handle)); ZipEntry entry; ZipEntry64 entry; ASSERT_EQ(0, FindEntry(handle, "a.txt", &entry)); VectorWriter writer; Loading @@ -1315,7 +1315,7 @@ TEST_F(Zip64ParseTest, extractWithDataDescriptor) { ZipArchiveHandle handle; ASSERT_EQ( 0, OpenArchiveFromMemory(zip_content_.data(), zip_content_.size(), "debug_zip64", &handle)); ZipEntry entry; ZipEntry64 entry; ASSERT_EQ(0, FindEntry(handle, "b.txt", &entry)); VectorWriter writer; Loading