Add deduplicate_entry_values mode to TableFlattener. (8b33e7ad) · Commits · e / os / android_frameworks_base

tools/aapt2/format/binary/ResEntryWriter.cpp

+44 −6

Original line number	Diff line number	Diff line
		@@ -222,19 +222,57 @@ int32_t WriteMapToBuffer(const FlatEntry* map_entry, BigBuffer* buffer) {
		return offset;
		}

		void WriteItemToPair(const FlatEntry* item_entry, ResEntryValuePair* out_pair) {
		static_assert(sizeof(ResEntryValuePair) == sizeof(ResTable_entry) + sizeof(Res_value),
		"ResEntryValuePair must not have padding between entry and value.");

		WriteEntry<ResTable_entry>(item_entry, &out_pair->entry);

		CHECK(ValueCast<Item>(item_entry->value)->Flatten(&out_pair->value)) << "flatten failed";
		out_pair->value.size = android::util::HostToDevice16(sizeof(out_pair->value));
		}

		int32_t SequentialResEntryWriter::WriteMap(const FlatEntry* entry) {
		return WriteMapToBuffer(entry, entries_buffer_);
		}

		int32_t SequentialResEntryWriter::WriteItem(const FlatEntry* entry) {
		int32_t offset = entries_buffer_->size();
		ResTable_entry* out_entry = entries_buffer_->NextBlock<ResTable_entry>();
		Res_value* out_value = entries_buffer_->NextBlock<Res_value>();
		out_value->size = android::util::HostToDevice16(sizeof(*out_value));

		WriteEntry<ResTable_entry>(entry, out_entry);
		CHECK(ValueCast<Item>(entry->value)->Flatten(out_value)) << "flatten failed";
		auto* out_pair = entries_buffer_->NextBlock<ResEntryValuePair>();
		WriteItemToPair(entry, out_pair);
		return offset;
		}

		std::size_t ResEntryValuePairContentHasher::operator()(const ResEntryValuePairRef& ref) const {
		return android::JenkinsHashMixBytes(0, ref.ptr, sizeof(ResEntryValuePair));
		}

		bool ResEntryValuePairContentEqualTo::operator()(const ResEntryValuePairRef& a,
		const ResEntryValuePairRef& b) const {
		return std::memcmp(a.ptr, b.ptr, sizeof(ResEntryValuePair)) == 0;
		}

		int32_t DeduplicateItemsResEntryWriter::WriteMap(const FlatEntry* entry) {
		return WriteMapToBuffer(entry, entries_buffer_);
		}

		int32_t DeduplicateItemsResEntryWriter::WriteItem(const FlatEntry* entry) {
		int32_t initial_offset = entries_buffer_->size();

		auto* out_pair = entries_buffer_->NextBlock<ResEntryValuePair>();
		WriteItemToPair(entry, out_pair);

		auto ref = ResEntryValuePairRef{*out_pair};
		auto [it, inserted] = entry_offsets.insert({ref, initial_offset});
		if (inserted) {
		// If inserted just return a new offset as this is a first time we store
		// this entry.
		return initial_offset;
		}
		// If not inserted this means that this is a duplicate, backup allocated block to the buffer
		// and return offset of previously stored entry.
		entries_buffer_->BackUp(sizeof(ResEntryValuePair));
		return it->second;
		}

		} // namespace aapt
		No newline at end of file

tools/aapt2/format/binary/ResEntryWriter.h

+56 −0

Original line number	Diff line number	Diff line
		@@ -17,10 +17,13 @@
		#ifndef AAPT_FORMAT_BINARY_RESENTRY_SERIALIZER_H
		#define AAPT_FORMAT_BINARY_RESENTRY_SERIALIZER_H

		#include <unordered_map>

		#include "ResourceTable.h"
		#include "ValueVisitor.h"
		#include "android-base/macros.h"
		#include "androidfw/BigBuffer.h"
		#include "androidfw/ResourceTypes.h"

		namespace aapt {

		@@ -32,6 +35,35 @@ struct FlatEntry {
		uint32_t entry_key;
		};

		// Pair of ResTable_entry and Res_value. These pairs are stored sequentially in values buffer.
		// We introduce this structure for ResEntryWriter to a have single allocation using
		// BigBuffer::NextBlock which allows to return it back with BigBuffer::Backup.
		struct ResEntryValuePair {
		android::ResTable_entry entry;
		android::Res_value value;
		};

		// References ResEntryValuePair object stored in BigBuffer used as a key in std::unordered_map.
		// Allows access to memory address where ResEntryValuePair is stored.
		union ResEntryValuePairRef {
		const std::reference_wrapper<const ResEntryValuePair> pair;
		const u_char* ptr;

		explicit ResEntryValuePairRef(const ResEntryValuePair& ref) : pair(ref) {
		}
		};

		// Hasher which computes hash of ResEntryValuePair using its bytes representation in memory.
		struct ResEntryValuePairContentHasher {
		std::size_t operator()(const ResEntryValuePairRef& ref) const;
		};

		// Equaler which compares ResEntryValuePairs using theirs bytes representation in memory.
		struct ResEntryValuePairContentEqualTo {
		bool operator()(const ResEntryValuePairRef& a, const ResEntryValuePairRef& b) const;
		};

		// Base class that allows to write FlatEntries into entries_buffer.
		class ResEntryWriter {
		public:
		virtual ~ResEntryWriter() = default;
		@@ -59,6 +91,8 @@ class ResEntryWriter {
		DISALLOW_COPY_AND_ASSIGN(ResEntryWriter);
		};

		// ResEntryWriter which writes FlatEntries sequentially into entries_buffer.
		// Next entry is always written right after previous one in the buffer.
		class SequentialResEntryWriter : public ResEntryWriter {
		public:
		explicit SequentialResEntryWriter(android::BigBuffer* entries_buffer)
		@@ -74,6 +108,28 @@ class SequentialResEntryWriter : public ResEntryWriter {
		DISALLOW_COPY_AND_ASSIGN(SequentialResEntryWriter);
		};

		// ResEntryWriter that writes only unique entry and value pairs into entries_buffer.
		// Next entry is written into buffer only if there is no entry with the same bytes representation
		// in memory written before. Otherwise returns offset of already written entry.
		class DeduplicateItemsResEntryWriter : public ResEntryWriter {
		public:
		explicit DeduplicateItemsResEntryWriter(android::BigBuffer* entries_buffer)
		: ResEntryWriter(entries_buffer) {
		}
		~DeduplicateItemsResEntryWriter() override = default;

		int32_t WriteItem(const FlatEntry* entry) override;

		int32_t WriteMap(const FlatEntry* entry) override;

		private:
		DISALLOW_COPY_AND_ASSIGN(DeduplicateItemsResEntryWriter);

		std::unordered_map<ResEntryValuePairRef, int32_t, ResEntryValuePairContentHasher,
		ResEntryValuePairContentEqualTo>
		entry_offsets;
		};

		} // namespace aapt

		#endif
		No newline at end of file

tools/aapt2/format/binary/ResEntryWriter_test.cpp

0 → 100644

+138 −0

Original line number	Diff line number	Diff line
		/*
		* Copyright (C) 2022 The Android Open Source Project
		*
		* Licensed under the Apache License, Version 2.0 (the "License");
		* you may not use this file except in compliance with the License.
		* You may obtain a copy of the License at
		*
		* http://www.apache.org/licenses/LICENSE-2.0
		*
		* Unless required by applicable law or agreed to in writing, software
		* distributed under the License is distributed on an "AS IS" BASIS,
		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
		* See the License for the specific language governing permissions and
		* limitations under the License.
		*/

		#include "format/binary/ResEntryWriter.h"

		#include "androidfw/BigBuffer.h"
		#include "format/binary/ResourceTypeExtensions.h"
		#include "test/Test.h"
		#include "util/Util.h"

		using ::android::BigBuffer;
		using ::android::Res_value;
		using ::android::ResTable_map;
		using ::testing::Eq;
		using ::testing::Ge;
		using ::testing::IsNull;
		using ::testing::Ne;
		using ::testing::NotNull;

		namespace aapt {

		using SequentialResEntryWriterTest = CommandTestFixture;
		using DeduplicateItemsResEntryWriterTest = CommandTestFixture;

		std::vector<int32_t> WriteAllEntries(const ResourceTableView& table, ResEntryWriter& writer) {
		std::vector<int32_t> result = {};
		for (const auto& type : table.packages[0].types) {
		for (const auto& entry : type.entries) {
		for (const auto& value : entry.values) {
		auto flat_entry = FlatEntry{&entry, value->value.get(), 0};
		result.push_back(writer.Write(&flat_entry));
		}
		}
		}
		return result;
		}

		TEST_F(SequentialResEntryWriterTest, WriteEntriesOneByOne) {
		std::unique_ptr<ResourceTable> table =
		test::ResourceTableBuilder()
		.AddSimple("com.app.test:id/id1", ResourceId(0x7f010000))
		.AddSimple("com.app.test:id/id2", ResourceId(0x7f010001))
		.AddSimple("com.app.test:id/id3", ResourceId(0x7f010002))
		.Build();

		BigBuffer out(512);
		SequentialResEntryWriter writer(&out);
		auto offsets = WriteAllEntries(table->GetPartitionedView(), writer);

		std::vector<int32_t> expected_offsets{0, sizeof(ResEntryValuePair),
		2 * sizeof(ResEntryValuePair)};
		EXPECT_EQ(out.size(), 3 * sizeof(ResEntryValuePair));
		EXPECT_EQ(offsets, expected_offsets);
		};

		TEST_F(SequentialResEntryWriterTest, WriteMapEntriesOneByOne) {
		std::unique_ptr<Array> array1 = util::make_unique<Array>();
		array1->elements.push_back(
		util::make_unique<BinaryPrimitive>(uint8_t(Res_value::TYPE_INT_DEC), 1u));
		array1->elements.push_back(
		util::make_unique<BinaryPrimitive>(uint8_t(Res_value::TYPE_INT_DEC), 2u));
		std::unique_ptr<Array> array2 = util::make_unique<Array>();
		array2->elements.push_back(
		util::make_unique<BinaryPrimitive>(uint8_t(Res_value::TYPE_INT_DEC), 1u));
		array2->elements.push_back(
		util::make_unique<BinaryPrimitive>(uint8_t(Res_value::TYPE_INT_DEC), 2u));

		std::unique_ptr<ResourceTable> table = test::ResourceTableBuilder()
		.AddValue("com.app.test:array/arr1", std::move(array1))
		.AddValue("com.app.test:array/arr2", std::move(array2))
		.Build();

		BigBuffer out(512);
		SequentialResEntryWriter writer(&out);
		auto offsets = WriteAllEntries(table->GetPartitionedView(), writer);

		std::vector<int32_t> expected_offsets{0, sizeof(ResTable_entry_ext) + 2 * sizeof(ResTable_map)};
		EXPECT_EQ(out.size(), 2 * (sizeof(ResTable_entry_ext) + 2 * sizeof(ResTable_map)));
		EXPECT_EQ(offsets, expected_offsets);
		};

		TEST_F(DeduplicateItemsResEntryWriterTest, DeduplicateItemEntries) {
		std::unique_ptr<ResourceTable> table =
		test::ResourceTableBuilder()
		.AddSimple("com.app.test:id/id1", ResourceId(0x7f010000))
		.AddSimple("com.app.test:id/id2", ResourceId(0x7f010001))
		.AddSimple("com.app.test:id/id3", ResourceId(0x7f010002))
		.Build();

		BigBuffer out(512);
		DeduplicateItemsResEntryWriter writer(&out);
		auto offsets = WriteAllEntries(table->GetPartitionedView(), writer);

		std::vector<int32_t> expected_offsets{0, 0, 0};
		EXPECT_EQ(out.size(), sizeof(ResEntryValuePair));
		EXPECT_EQ(offsets, expected_offsets);
		};

		TEST_F(DeduplicateItemsResEntryWriterTest, WriteMapEntriesOneByOne) {
		std::unique_ptr<Array> array1 = util::make_unique<Array>();
		array1->elements.push_back(
		util::make_unique<BinaryPrimitive>(uint8_t(Res_value::TYPE_INT_DEC), 1u));
		array1->elements.push_back(
		util::make_unique<BinaryPrimitive>(uint8_t(Res_value::TYPE_INT_DEC), 2u));
		std::unique_ptr<Array> array2 = util::make_unique<Array>();
		array2->elements.push_back(
		util::make_unique<BinaryPrimitive>(uint8_t(Res_value::TYPE_INT_DEC), 1u));
		array2->elements.push_back(
		util::make_unique<BinaryPrimitive>(uint8_t(Res_value::TYPE_INT_DEC), 2u));

		std::unique_ptr<ResourceTable> table = test::ResourceTableBuilder()
		.AddValue("com.app.test:array/arr1", std::move(array1))
		.AddValue("com.app.test:array/arr2", std::move(array2))
		.Build();

		BigBuffer out(512);
		DeduplicateItemsResEntryWriter writer(&out);
		auto offsets = WriteAllEntries(table->GetPartitionedView(), writer);

		std::vector<int32_t> expected_offsets{0, sizeof(ResTable_entry_ext) + 2 * sizeof(ResTable_map)};
		EXPECT_EQ(out.size(), 2 * (sizeof(ResTable_entry_ext) + 2 * sizeof(ResTable_map)));
		EXPECT_EQ(offsets, expected_offsets);
		};

		} // namespace aapt
		No newline at end of file

tools/aapt2/format/binary/TableFlattener.cpp

+18 −8

Original line number	Diff line number	Diff line
		@@ -16,15 +16,13 @@

		#include "format/binary/TableFlattener.h"

		#include <algorithm>
		#include <numeric>
		#include <sstream>
		#include <type_traits>
		#include <variant>

		#include "ResourceTable.h"
		#include "ResourceValues.h"
		#include "SdkConstants.h"
		#include "ValueVisitor.h"
		#include "android-base/logging.h"
		#include "android-base/macros.h"
		#include "android-base/stringprintf.h"
		@@ -70,14 +68,16 @@ class PackageFlattener {
		PackageFlattener(IAaptContext* context, const ResourceTablePackageView& package,
		const std::map<size_t, std::string>* shared_libs,
		SparseEntriesMode sparse_entries, bool collapse_key_stringpool,
		const std::set<ResourceName>& name_collapse_exemptions)
		const std::set<ResourceName>& name_collapse_exemptions,
		bool deduplicate_entry_values)
		: context_(context),
		diag_(context->GetDiagnostics()),
		package_(package),
		shared_libs_(shared_libs),
		sparse_entries_(sparse_entries),
		collapse_key_stringpool_(collapse_key_stringpool),
		name_collapse_exemptions_(name_collapse_exemptions) {
		name_collapse_exemptions_(name_collapse_exemptions),
		deduplicate_entry_values_(deduplicate_entry_values) {
		}

		bool FlattenPackage(BigBuffer* buffer) {
		@@ -151,10 +151,18 @@ class PackageFlattener {
		offsets.resize(num_total_entries, 0xffffffffu);

		android::BigBuffer values_buffer(512);
		SequentialResEntryWriter res_entry_writer(&values_buffer);
		std::variant<std::monostate, DeduplicateItemsResEntryWriter, SequentialResEntryWriter>
		writer_variant;
		ResEntryWriter* res_entry_writer;
		if (deduplicate_entry_values_) {
		res_entry_writer = &writer_variant.emplace<DeduplicateItemsResEntryWriter>(&values_buffer);
		} else {
		res_entry_writer = &writer_variant.emplace<SequentialResEntryWriter>(&values_buffer);
		}

		for (FlatEntry& flat_entry : *entries) {
		CHECK(static_cast<size_t>(flat_entry.entry->id.value()) < num_total_entries);
		offsets[flat_entry.entry->id.value()] = res_entry_writer.Write(&flat_entry);
		offsets[flat_entry.entry->id.value()] = res_entry_writer->Write(&flat_entry);
		}

		bool sparse_encode = sparse_entries_ == SparseEntriesMode::Enabled \|\|
		@@ -510,6 +518,7 @@ class PackageFlattener {
		bool collapse_key_stringpool_;
		const std::set<ResourceName>& name_collapse_exemptions_;
		std::map<uint32_t, uint32_t> aliases_;
		bool deduplicate_entry_values_;
		};

		} // namespace
		@@ -561,7 +570,8 @@ bool TableFlattener::Consume(IAaptContext* context, ResourceTable* table) {

		PackageFlattener flattener(context, package, &table->included_packages_,
		options_.sparse_entries, options_.collapse_key_stringpool,
		options_.name_collapse_exemptions);
		options_.name_collapse_exemptions,
		options_.deduplicate_entry_values);
		if (!flattener.FlattenPackage(&package_buffer)) {
		return false;
		}

tools/aapt2/format/binary/TableFlattener.h

+14 −0

Original line number	Diff line number	Diff line
		@@ -54,6 +54,20 @@ struct TableFlattenerOptions {

		// Map from original resource paths to shortened resource paths.
		std::map<std::string, std::string> shortened_path_map;

		// When enabled, only unique pairs of entry and value are stored in type chunks.
		//
		// By default, all such pairs are unique because a reference to resource name in the string pool
		// is a part of the pair. But when resource names are collapsed (using 'collapse_key_stringpool'
		// flag or manually) the same data might be duplicated multiple times in the same type chunk.
		//
		// For example: an application has 3 boolean resources with collapsed names and 3 'true' values
		// are defined for these resources in 'default' configuration. All pairs of entry and value for
		// these resources will have the same binary representation and stored only once in type chunk
		// instead of three times when this flag is disabled.
		//
		// This applies only to simple entries (entry->flags & ResTable_entry::FLAG_COMPLEX == 0).
		bool deduplicate_entry_values = false;
		};

		class TableFlattener : public IResourceTableConsumer {