Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 9a8dbf26 authored by Jeremy Meyer's avatar Jeremy Meyer Committed by Android (Google) Code Review
Browse files

Merge "Store string frros in a string pool and copy that pool to the idmap"

parents 33979d57 6587102e

cmds/idmap2/include/idmap2/FabricatedOverlay.h

+7 −4
Original line number Diff line number Diff line
@@ -66,18 +66,21 @@ struct FabricatedOverlay { 


 private:

  struct SerializedData {

    std::unique_ptr<uint8_t[]> data;

    size_t data_size;

    uint32_t crc;

    std::unique_ptr<uint8_t[]> pb_data;

    size_t pb_data_size;

    uint32_t pb_crc;

    std::string sp_data;

   };



  Result<SerializedData*> InitializeData() const;

  Result<uint32_t> GetCrc() const;



  explicit FabricatedOverlay(pb::FabricatedOverlay&& overlay,

                             std::string&& string_pool_data_,

                             std::optional<uint32_t> crc_from_disk = {});



  pb::FabricatedOverlay overlay_pb_;

  std::string string_pool_data_;

  std::optional<uint32_t> crc_from_disk_;

  mutable std::optional<SerializedData> data_;

cmds/idmap2/libidmap2/FabricatedOverlay.cpp

+78 −30
Original line number Diff line number Diff line
@@ -17,8 +17,9 @@ 
#include "idmap2/FabricatedOverlay.h"



#include <androidfw/ResourceUtils.h>

#include <androidfw/StringPool.h>

#include <google/protobuf/io/coded_stream.h>

#include <google/protobuf/io/zero_copy_stream_impl_lite.h>

#include <google/protobuf/io/zero_copy_stream_impl.h>

#include <utils/ByteOrder.h>

#include <zlib.h>
@@ -30,6 +31,8 @@ 


namespace android::idmap2 {



constexpr auto kBufferSize = 1024;



namespace {

bool Read32(std::istream& stream, uint32_t* out) {

  uint32_t value;
@@ -47,8 +50,11 @@ void Write32(std::ostream& stream, uint32_t value) { 
}  // namespace



FabricatedOverlay::FabricatedOverlay(pb::FabricatedOverlay&& overlay,

                                     std::string&& string_pool_data,

                                     std::optional<uint32_t> crc_from_disk)

    : overlay_pb_(std::forward<pb::FabricatedOverlay>(overlay)), crc_from_disk_(crc_from_disk) {

    : overlay_pb_(std::forward<pb::FabricatedOverlay>(overlay)),

    string_pool_data_(std::move(string_pool_data)),

    crc_from_disk_(crc_from_disk) {

}



FabricatedOverlay::Builder::Builder(const std::string& package_name, const std::string& name,
@@ -76,7 +82,11 @@ FabricatedOverlay::Builder& FabricatedOverlay::Builder::SetResourceValue( 
}



Result<FabricatedOverlay> FabricatedOverlay::Builder::Build() {

  std::map<std::string, std::map<std::string, std::map<std::string, TargetValue>>> entries;

  using EntryMap = std::map<std::string, TargetValue>;

  using TypeMap = std::map<std::string, EntryMap>;

  using PackageMap = std::map<std::string, TypeMap>;

  PackageMap package_map;

  android::StringPool string_pool;

  for (const auto& res_entry : entries_) {

    StringPiece package_substr;

    StringPiece type_name;
@@ -96,11 +106,10 @@ Result<FabricatedOverlay> FabricatedOverlay::Builder::Build() { 
      return Error("resource name '%s' missing entry name", res_entry.resource_name.c_str());

    }



    auto package = entries.find(package_name);

    if (package == entries.end()) {

      package = entries

                    .insert(std::make_pair(

                        package_name, std::map<std::string, std::map<std::string, TargetValue>>()))

    auto package = package_map.find(package_name);

    if (package == package_map.end()) {

      package = package_map

                    .insert(std::make_pair(package_name, TypeMap()))

                    .first;

    }
@@ -108,7 +117,7 @@ Result<FabricatedOverlay> FabricatedOverlay::Builder::Build() { 
    if (type == package->second.end()) {

      type =

          package->second

              .insert(std::make_pair(type_name.to_string(), std::map<std::string, TargetValue>()))

              .insert(std::make_pair(type_name.to_string(), EntryMap()))

              .first;

    }
@@ -127,25 +136,32 @@ Result<FabricatedOverlay> FabricatedOverlay::Builder::Build() { 
  overlay_pb.set_target_package_name(target_package_name_);

  overlay_pb.set_target_overlayable(target_overlayable_);



  for (const auto& package : entries) {

  for (auto& package : package_map) {

    auto package_pb = overlay_pb.add_packages();

    package_pb->set_name(package.first);



    for (const auto& type : package.second) {

    for (auto& type : package.second) {

      auto type_pb = package_pb->add_types();

      type_pb->set_name(type.first);



      for (const auto& entry : type.second) {

      for (auto& entry : type.second) {

        auto entry_pb = type_pb->add_entries();

        entry_pb->set_name(entry.first);

        pb::ResourceValue* value = entry_pb->mutable_res_value();

        value->set_data_type(entry.second.data_type);

        if (entry.second.data_type == Res_value::TYPE_STRING) {

          auto ref = string_pool.MakeRef(entry.second.data_string_value);

          value->set_data_value(ref.index());

        } else {

          value->set_data_value(entry.second.data_value);

        }

      }

    }

  }



  return FabricatedOverlay(std::move(overlay_pb));

  android::BigBuffer string_buffer(kBufferSize);

  android::StringPool::FlattenUtf8(&string_buffer, string_pool, nullptr);

  return FabricatedOverlay(std::move(overlay_pb), string_buffer.to_string());

}



Result<FabricatedOverlay> FabricatedOverlay::FromBinaryStream(std::istream& stream) {
@@ -163,48 +179,67 @@ Result<FabricatedOverlay> FabricatedOverlay::FromBinaryStream(std::istream& stre 
    return Error("Failed to read fabricated overlay version.");

  }



  if (version != 1) {

  if (version != 1 && version != 2) {

    return Error("Invalid fabricated overlay version '%u'.", version);

  }



  uint32_t crc;

  if (!Read32(stream, &crc)) {

    return Error("Failed to read fabricated overlay version.");

    return Error("Failed to read fabricated overlay crc.");

  }



  pb::FabricatedOverlay overlay{};

  std::string sp_data;

  if (version == 2) {

    uint32_t sp_size;

    if (!Read32(stream, &sp_size)) {

      return Error("Failed read string pool size.");

    }

    std::string buf(sp_size, '\0');

    if (!stream.read(buf.data(), sp_size)) {

      return Error("Failed to read string pool.");

    }

    sp_data = buf;



    if (!overlay.ParseFromIstream(&stream)) {

      return Error("Failed read fabricated overlay proto.");

    }

  } else {

    if (!overlay.ParseFromIstream(&stream)) {

      return Error("Failed read fabricated overlay proto.");

    }

  }



  // If the proto version is the latest version, then the contents of the proto must be the same

  // when the proto is re-serialized; otherwise, the crc must be calculated because migrating the

  // proto to the latest version will likely change the contents of the fabricated overlay.

  return FabricatedOverlay(std::move(overlay), version == kFabricatedOverlayCurrentVersion

  return FabricatedOverlay(std::move(overlay), std::move(sp_data),

                           version == kFabricatedOverlayCurrentVersion

                                                   ? std::optional<uint32_t>(crc)

                                                   : std::nullopt);

}



Result<FabricatedOverlay::SerializedData*> FabricatedOverlay::InitializeData() const {

  if (!data_.has_value()) {

    auto size = overlay_pb_.ByteSizeLong();

    auto data = std::unique_ptr<uint8_t[]>(new uint8_t[size]);

    auto pb_size = overlay_pb_.ByteSizeLong();

    auto pb_data = std::unique_ptr<uint8_t[]>(new uint8_t[pb_size]);



    // Ensure serialization is deterministic

    google::protobuf::io::ArrayOutputStream array_stream(data.get(), size);

    google::protobuf::io::ArrayOutputStream array_stream(pb_data.get(), pb_size);

    google::protobuf::io::CodedOutputStream output_stream(&array_stream);

    output_stream.SetSerializationDeterministic(true);

    overlay_pb_.SerializeWithCachedSizes(&output_stream);

    if (output_stream.HadError() || size != output_stream.ByteCount()) {

    if (output_stream.HadError() || pb_size != output_stream.ByteCount()) {

      return Error("Failed to serialize fabricated overlay.");

    }



    // Calculate the crc using the proto data and the version.

    uint32_t crc = crc32(0L, Z_NULL, 0);

    crc = crc32(crc, reinterpret_cast<const uint8_t*>(&kFabricatedOverlayCurrentVersion),

    uint32_t pb_crc = crc32(0L, Z_NULL, 0);

    pb_crc = crc32(pb_crc, reinterpret_cast<const uint8_t*>(&kFabricatedOverlayCurrentVersion),

                sizeof(uint32_t));

    crc = crc32(crc, data.get(), size);

    data_ = SerializedData{std::move(data), size, crc};

    pb_crc = crc32(pb_crc, pb_data.get(), pb_size);



    data_ = SerializedData{std::move(pb_data), pb_size, pb_crc, string_pool_data_};

  }

  return &(*data_);

}
@@ -216,7 +251,7 @@ Result<uint32_t> FabricatedOverlay::GetCrc() const { 
  if (!data) {

    return data.GetError();

  }

  return (*data)->crc;

  return (*data)->pb_crc;

}



Result<Unit> FabricatedOverlay::ToBinaryStream(std::ostream& stream) const {
@@ -227,8 +262,13 @@ Result<Unit> FabricatedOverlay::ToBinaryStream(std::ostream& stream) const { 


  Write32(stream, kFabricatedOverlayMagic);

  Write32(stream, kFabricatedOverlayCurrentVersion);

  Write32(stream, (*data)->crc);

  stream.write(reinterpret_cast<const char*>((*data)->data.get()), (*data)->data_size);

  Write32(stream, (*data)->pb_crc);

  Write32(stream, (*data)->sp_data.length());

  stream.write((*data)->sp_data.data(), (*data)->sp_data.length());

  if (stream.bad()) {

    return Error("Failed to write string pool data.");

  }

  stream.write(reinterpret_cast<const char*>((*data)->pb_data.get()), (*data)->pb_data_size);

  if (stream.bad()) {

    return Error("Failed to write serialized fabricated overlay.");

  }
@@ -295,6 +335,14 @@ Result<OverlayData> FabContainer::GetOverlayData(const OverlayManifestInfo& info 
      }

    }

  }

  const uint32_t string_pool_data_length = overlay_.string_pool_data_.length();

  result.string_pool_data = OverlayData::InlineStringPoolData{

      .data = std::unique_ptr<uint8_t[]>(new uint8_t[string_pool_data_length]),

      .data_length = string_pool_data_length,

      .string_pool_offset = 0,

  };

  memcpy(result.string_pool_data->data.get(), overlay_.string_pool_data_.data(),

       string_pool_data_length);

  return result;

}

cmds/idmap2/tests/FabricatedOverlayTests.cpp

+22 −3
Original line number Diff line number Diff line
@@ -46,6 +46,7 @@ TEST(FabricatedOverlayTests, SetResourceValue) { 
          .SetResourceValue("com.example.target:integer/int1", Res_value::TYPE_INT_DEC, 1U)

          .SetResourceValue("com.example.target.split:integer/int2", Res_value::TYPE_INT_DEC, 2U)

          .SetResourceValue("string/int3", Res_value::TYPE_REFERENCE, 0x7f010000)

          .SetResourceValue("com.example.target:string/string1", Res_value::TYPE_STRING, "foobar")

          .Build();

  ASSERT_TRUE(overlay);

  auto container = FabricatedOverlayContainer::FromOverlay(std::move(*overlay));
@@ -59,8 +60,9 @@ TEST(FabricatedOverlayTests, SetResourceValue) { 


  auto pairs = container->GetOverlayData(*info);

  ASSERT_TRUE(pairs);

  EXPECT_FALSE(pairs->string_pool_data.has_value());

  ASSERT_EQ(3U, pairs->pairs.size());

  ASSERT_EQ(4U, pairs->pairs.size());

  auto string_pool = ResStringPool(pairs->string_pool_data->data.get(),

                                        pairs->string_pool_data->data_length, false);



  auto& it = pairs->pairs[0];

  ASSERT_EQ("com.example.target:integer/int1", it.resource_name);
@@ -77,6 +79,13 @@ TEST(FabricatedOverlayTests, SetResourceValue) { 
  ASSERT_EQ(Res_value::TYPE_REFERENCE, entry->data_type);



  it = pairs->pairs[2];

  ASSERT_EQ("com.example.target:string/string1", it.resource_name);

  entry = std::get_if<TargetValue>(&it.value);

  ASSERT_NE(nullptr, entry);

  ASSERT_EQ(Res_value::TYPE_STRING, entry->data_type);

  ASSERT_EQ(std::string("foobar"), string_pool.string8At(entry->data_value).value_or(""));



  it = pairs->pairs[3];

  ASSERT_EQ("com.example.target.split:integer/int2", it.resource_name);

  entry = std::get_if<TargetValue>(&it.value);

  ASSERT_NE(nullptr, entry);
@@ -104,6 +113,7 @@ TEST(FabricatedOverlayTests, SerializeAndDeserialize) { 
      FabricatedOverlay::Builder("com.example.overlay", "SandTheme", "com.example.target")

          .SetOverlayable("TestResources")

          .SetResourceValue("com.example.target:integer/int1", Res_value::TYPE_INT_DEC, 1U)

          .SetResourceValue("com.example.target:string/string1", Res_value::TYPE_STRING, "foobar")

          .Build();

  ASSERT_TRUE(overlay);

  TemporaryFile tf;
@@ -126,7 +136,9 @@ TEST(FabricatedOverlayTests, SerializeAndDeserialize) { 


  auto pairs = (*container)->GetOverlayData(*info);

  ASSERT_TRUE(pairs) << pairs.GetErrorMessage();

  EXPECT_EQ(1U, pairs->pairs.size());

  EXPECT_EQ(2U, pairs->pairs.size());

  auto string_pool = ResStringPool(pairs->string_pool_data->data.get(),

                                        pairs->string_pool_data->data_length, false);



  auto& it = pairs->pairs[0];

  ASSERT_EQ("com.example.target:integer/int1", it.resource_name);
@@ -134,6 +146,13 @@ TEST(FabricatedOverlayTests, SerializeAndDeserialize) { 
  ASSERT_NE(nullptr, entry);

  EXPECT_EQ(1U, entry->data_value);

  EXPECT_EQ(Res_value::TYPE_INT_DEC, entry->data_type);



  it = pairs->pairs[1];

  ASSERT_EQ("com.example.target:string/string1", it.resource_name);

  entry = std::get_if<TargetValue>(&it.value);

  ASSERT_NE(nullptr, entry);

  ASSERT_EQ(Res_value::TYPE_STRING, entry->data_type);

  ASSERT_EQ(std::string("foobar"), string_pool.string8At(entry->data_value).value_or(""));

}



}  // namespace android::idmap2

cmds/idmap2/tests/IdmapTests.cpp

+9 −1
Original line number Diff line number Diff line
@@ -263,6 +263,7 @@ TEST(IdmapTests, FabricatedOverlay) { 
                  .SetOverlayable("TestResources")

                  .SetResourceValue("integer/int1", Res_value::TYPE_INT_DEC, 2U)

                  .SetResourceValue("string/str1", Res_value::TYPE_REFERENCE, 0x7f010000)

                  .SetResourceValue("string/str2", Res_value::TYPE_STRING, "foobar")

                  .Build();



  ASSERT_TRUE(frro);
@@ -288,12 +289,19 @@ TEST(IdmapTests, FabricatedOverlay) { 
  ASSERT_EQ(data->GetTargetEntries().size(), 0U);

  ASSERT_EQ(data->GetOverlayEntries().size(), 0U);



  auto string_pool_data = data->GetStringPoolData();

  auto string_pool = ResStringPool(string_pool_data.data(), string_pool_data.size(), false);





  const auto& target_inline_entries = data->GetTargetInlineEntries();

  ASSERT_EQ(target_inline_entries.size(), 2U);

  ASSERT_EQ(target_inline_entries.size(), 3U);

  ASSERT_TARGET_INLINE_ENTRY(target_inline_entries[0], R::target::integer::int1,

                             Res_value::TYPE_INT_DEC, 2U);

  ASSERT_TARGET_INLINE_ENTRY(target_inline_entries[1], R::target::string::str1,

                             Res_value::TYPE_REFERENCE, 0x7f010000);

  ASSERT_TARGET_INLINE_ENTRY(target_inline_entries[2], R::target::string::str2,

                             Res_value::TYPE_STRING,

                             (uint32_t) (string_pool.indexOfString(u"foobar", 6)).value_or(-1));

}



TEST(IdmapTests, FailCreateIdmapInvalidName) {

cmds/idmap2/tests/R.h

+1 −0
Original line number Diff line number Diff line
@@ -41,6 +41,7 @@ namespace R::target { 
    constexpr ResourceId policy_system = 0x7f02000c;

    constexpr ResourceId policy_system_vendor = 0x7f02000d;

    constexpr ResourceId str1 = 0x7f02000e;

    constexpr ResourceId str2 = 0x7f02000f;

    constexpr ResourceId str3 = 0x7f020010;

    constexpr ResourceId str4 = 0x7f020011;

  }  // namespace string