Merge "Pseudolocale support for grammatical gender" into main (40e10062) · Commits · e / os / android_frameworks_base

tools/aapt2/cmd/Compile.cpp

+14 −1

Original line number	Diff line number	Diff line
		@@ -186,7 +186,20 @@ static bool CompileTable(IAaptContext* context, const CompileOptions& options,
		// These are created as weak symbols, and are only generated from default
		// configuration
		// strings and plurals.
		PseudolocaleGenerator pseudolocale_generator;
		std::string grammatical_gender_values;
		std::string grammatical_gender_ratio;
		if (options.pseudo_localize_gender_values) {
		grammatical_gender_values = options.pseudo_localize_gender_values.value();
		} else {
		grammatical_gender_values = "f,m,n";
		}
		if (options.pseudo_localize_gender_ratio) {
		grammatical_gender_ratio = options.pseudo_localize_gender_ratio.value();
		} else {
		grammatical_gender_ratio = "1.0";
		}
		PseudolocaleGenerator pseudolocale_generator(grammatical_gender_values,
		grammatical_gender_ratio);
		if (!pseudolocale_generator.Consume(context, &table)) {
		return false;
		}

tools/aapt2/cmd/Compile.h

+11 −0

Original line number	Diff line number	Diff line
		@@ -35,6 +35,8 @@ struct CompileOptions {
		std::optional<std::string> res_dir;
		std::optional<std::string> res_zip;
		std::optional<std::string> generate_text_symbols_path;
		std::optional<std::string> pseudo_localize_gender_values;
		std::optional<std::string> pseudo_localize_gender_ratio;
		std::optional<Visibility::Level> visibility;
		bool pseudolocalize = false;
		bool no_png_crunch = false;
		@@ -76,6 +78,15 @@ class CompileCommand : public Command {
		AddOptionalFlag("--source-path",
		"Sets the compiled resource file source file path to the given string.",
		&options_.source_path);
		AddOptionalFlag("--pseudo-localize-gender-values",
		"Sets the gender values to pick up for generating grammatical gender strings, "
		"gender values should be f, m, or n, which are shortcuts for feminine, "
		"masculine and neuter, and split with comma.",
		&options_.pseudo_localize_gender_values);
		AddOptionalFlag("--pseudo-localize-gender-ratio",
		"Sets the ratio of resources to generate grammatical gender strings for. The "
		"ratio has to be a float number between 0 and 1.",
		&options_.pseudo_localize_gender_ratio);
		}

		int Action(const std::vector<std::string>& args) override;

tools/aapt2/cmd/Compile_test.cpp

+18 −3

Original line number	Diff line number	Diff line
		@@ -236,9 +236,24 @@ TEST_F(CompilerTest, DoNotTranslateTest) {
		// The first string (000) is translatable, the second is not
		// ar-XB uses "\u200F\u202E...\u202C\u200F"
		std::vector<std::string> expected_translatable = {
		"000", "111", // default locale
		"(F)[000 one]", // en-XA-feminine
		"(F)\xE2\x80\x8F\xE2\x80\xAE"
		"000"
		"\xE2\x80\xAC\xE2\x80\x8F", // ar-XB-feminine
		"(M)[000 one]", // en-XA-masculine
		"(M)\xE2\x80\x8F\xE2\x80\xAE"
		"000"
		"\xE2\x80\xAC\xE2\x80\x8F", // ar-XB-masculine
		"(N)[000 one]", // en-XA-neuter
		"(N)\xE2\x80\x8F\xE2\x80\xAE"
		"000"
		"\xE2\x80\xAC\xE2\x80\x8F", // ar-XB-neuter
		"000", // default locale
		"111", // default locale
		"[000 one]", // en-XA
		"\xE2\x80\x8F\xE2\x80\xAE" "000" "\xE2\x80\xAC\xE2\x80\x8F", // ar-XB
		"\xE2\x80\x8F\xE2\x80\xAE"
		"000"
		"\xE2\x80\xAC\xE2\x80\x8F", // ar-XB
		};
		AssertTranslations(this, "foo", expected_translatable);
		AssertTranslations(this, "foo_donottranslate", expected_translatable);

tools/aapt2/compile/PseudolocaleGenerator.cpp

+197 −6

Original line number	Diff line number	Diff line
		@@ -16,11 +16,15 @@

		#include "compile/PseudolocaleGenerator.h"

		#include <stdint.h>

		#include <algorithm>
		#include <random>

		#include "ResourceTable.h"
		#include "ResourceValues.h"
		#include "ValueVisitor.h"
		#include "androidfw/ResourceTypes.h"
		#include "androidfw/Util.h"
		#include "compile/Pseudolocalizer.h"
		#include "util/Util.h"
		@@ -293,8 +297,85 @@ class Visitor : public ValueVisitor {
		Pseudolocalizer localizer_;
		};

		class GrammaticalGenderVisitor : public ValueVisitor {
		public:
		std::unique_ptr<Value> value;
		std::unique_ptr<Item> item;

		GrammaticalGenderVisitor(android::StringPool* pool, uint8_t grammaticalInflection)
		: pool_(pool), grammaticalInflection_(grammaticalInflection) {
		}

		void Visit(Plural* plural) override {
		CloningValueTransformer cloner(pool_);
		std::unique_ptr<Plural> grammatical_gendered = util::make_unique<Plural>();
		for (size_t i = 0; i < plural->values.size(); i++) {
		if (plural->values[i]) {
		GrammaticalGenderVisitor sub_visitor(pool_, grammaticalInflection_);
		plural->values[i]->Accept(&sub_visitor);
		if (sub_visitor.item) {
		grammatical_gendered->values[i] = std::move(sub_visitor.item);
		} else {
		grammatical_gendered->values[i] = plural->values[i]->Transform(cloner);
		}
		}
		}
		grammatical_gendered->SetSource(plural->GetSource());
		grammatical_gendered->SetWeak(true);
		value = std::move(grammatical_gendered);
		}

		std::string AddGrammaticalGenderPrefix(const std::string_view& original_string) {
		std::string result;
		switch (grammaticalInflection_) {
		case android::ResTable_config::GRAMMATICAL_GENDER_MASCULINE:
		result = std::string("(M)") + std::string(original_string);
		break;
		case android::ResTable_config::GRAMMATICAL_GENDER_FEMININE:
		result = std::string("(F)") + std::string(original_string);
		break;
		case android::ResTable_config::GRAMMATICAL_GENDER_NEUTER:
		result = std::string("(N)") + std::string(original_string);
		break;
		default:
		result = std::string(original_string);
		break;
		}
		return result;
		}

		void Visit(String* string) override {
		std::string prefixed_string = AddGrammaticalGenderPrefix(std::string(*string->value));
		std::unique_ptr<String> grammatical_gendered =
		util::make_unique<String>(pool_->MakeRef(prefixed_string));
		grammatical_gendered->SetSource(string->GetSource());
		grammatical_gendered->SetWeak(true);
		item = std::move(grammatical_gendered);
		}

		void Visit(StyledString* string) override {
		std::string prefixed_string = AddGrammaticalGenderPrefix(std::string(string->value->value));
		android::StyleString new_string;
		new_string.str = std::move(prefixed_string);
		for (const android::StringPool::Span& span : string->value->spans) {
		new_string.spans.emplace_back(android::Span{*span.name, span.first_char, span.last_char});
		}
		std::unique_ptr<StyledString> grammatical_gendered =
		util::make_unique<StyledString>(pool_->MakeRef(new_string));
		grammatical_gendered->SetSource(string->GetSource());
		grammatical_gendered->SetWeak(true);
		item = std::move(grammatical_gendered);
		}

		private:
		DISALLOW_COPY_AND_ASSIGN(GrammaticalGenderVisitor);
		android::StringPool* pool_;
		uint8_t grammaticalInflection_;
		};

		ConfigDescription ModifyConfigForPseudoLocale(const ConfigDescription& base,
		Pseudolocalizer::Method m) {
		Pseudolocalizer::Method m,
		uint8_t grammaticalInflection) {
		ConfigDescription modified = base;
		switch (m) {
		case Pseudolocalizer::Method::kAccent:
		@@ -313,12 +394,64 @@ ConfigDescription ModifyConfigForPseudoLocale(const ConfigDescription& base,
		default:
		break;
		}
		modified.grammaticalInflection = grammaticalInflection;
		return modified;
		}

		void GrammaticalGender(ResourceConfigValue* original_value,
		ResourceConfigValue* localized_config_value, android::StringPool* pool,
		ResourceEntry* entry, const Pseudolocalizer::Method method,
		uint8_t grammaticalInflection) {
		GrammaticalGenderVisitor visitor(pool, grammaticalInflection);
		localized_config_value->value->Accept(&visitor);

		std::unique_ptr<Value> grammatical_gendered_value;
		if (visitor.value) {
		grammatical_gendered_value = std::move(visitor.value);
		} else if (visitor.item) {
		grammatical_gendered_value = std::move(visitor.item);
		}
		if (!grammatical_gendered_value) {
		return;
		}

		ConfigDescription config =
		ModifyConfigForPseudoLocale(original_value->config, method, grammaticalInflection);

		ResourceConfigValue* grammatical_gendered_config_value =
		entry->FindOrCreateValue(config, original_value->product);
		if (!grammatical_gendered_config_value->value) {
		// Only use auto-generated pseudo-localization if none is defined.
		grammatical_gendered_config_value->value = std::move(grammatical_gendered_value);
		}
		}

		const uint32_t MASK_MASCULINE = 1; // Bit mask for masculine
		const uint32_t MASK_FEMININE = 2; // Bit mask for feminine
		const uint32_t MASK_NEUTER = 4; // Bit mask for neuter

		void GrammaticalGenderIfNeeded(ResourceConfigValue* original_value, ResourceConfigValue* new_value,
		android::StringPool* pool, ResourceEntry* entry,
		const Pseudolocalizer::Method method, uint32_t gender_state) {
		if (gender_state & MASK_FEMININE) {
		GrammaticalGender(original_value, new_value, pool, entry, method,
		android::ResTable_config::GRAMMATICAL_GENDER_FEMININE);
		}

		if (gender_state & MASK_MASCULINE) {
		GrammaticalGender(original_value, new_value, pool, entry, method,
		android::ResTable_config::GRAMMATICAL_GENDER_MASCULINE);
		}

		if (gender_state & MASK_NEUTER) {
		GrammaticalGender(original_value, new_value, pool, entry, method,
		android::ResTable_config::GRAMMATICAL_GENDER_NEUTER);
		}
		}

		void PseudolocalizeIfNeeded(const Pseudolocalizer::Method method,
		ResourceConfigValue* original_value, android::StringPool* pool,
		ResourceEntry* entry) {
		ResourceEntry* entry, uint32_t gender_state, bool gender_flag) {
		Visitor visitor(pool, method);
		original_value->value->Accept(&visitor);

		@@ -333,8 +466,8 @@ void PseudolocalizeIfNeeded(const Pseudolocalizer::Method method,
		return;
		}

		ConfigDescription config_with_accent =
		ModifyConfigForPseudoLocale(original_value->config, method);
		ConfigDescription config_with_accent = ModifyConfigForPseudoLocale(
		original_value->config, method, android::ResTable_config::GRAMMATICAL_GENDER_ANY);

		ResourceConfigValue* new_config_value =
		entry->FindOrCreateValue(config_with_accent, original_value->product);
		@@ -342,6 +475,9 @@ void PseudolocalizeIfNeeded(const Pseudolocalizer::Method method,
		// Only use auto-generated pseudo-localization if none is defined.
		new_config_value->value = std::move(localized_value);
		}
		if (gender_flag) {
		GrammaticalGenderIfNeeded(original_value, new_config_value, pool, entry, method, gender_state);
		}
		}

		// A value is pseudolocalizable if it does not define a locale (or is the default locale) and is
		@@ -356,16 +492,71 @@ static bool IsPseudolocalizable(ResourceConfigValue* config_value) {

		} // namespace

		bool ParseGenderValuesAndSaveState(const std::string& grammatical_gender_values,
		uint32_t* gender_state, android::IDiagnostics* diag) {
		std::vector<std::string> values = util::SplitAndLowercase(grammatical_gender_values, ',');
		for (size_t i = 0; i < values.size(); i++) {
		if (values[i].length() != 0) {
		if (values[i] == "f") {
		*gender_state \|= MASK_FEMININE;
		} else if (values[i] == "m") {
		*gender_state \|= MASK_MASCULINE;
		} else if (values[i] == "n") {
		*gender_state \|= MASK_NEUTER;
		} else {
		diag->Error(android::DiagMessage() << "Invalid grammatical gender value: " << values[i]);
		return false;
		}
		}
		}
		return true;
		}

		bool ParseGenderRatio(const std::string& grammatical_gender_ratio, float* gender_ratio,
		android::IDiagnostics* diag) {
		const char* input = grammatical_gender_ratio.c_str();
		char* endPtr;
		errno = 0;
		*gender_ratio = strtof(input, &endPtr);
		if (endPtr == input \|\| endPtr != '\0' \|\| errno == ERANGE \|\| gender_ratio < 0 \|\|
		*gender_ratio > 1) {
		diag->Error(android::DiagMessage()
		<< "Invalid grammatical gender ratio: " << grammatical_gender_ratio
		<< ", must be a real number between 0 and 1");
		return false;
		}
		return true;
		}

		bool PseudolocaleGenerator::Consume(IAaptContext* context, ResourceTable* table) {
		uint32_t gender_state = 0;
		if (!ParseGenderValuesAndSaveState(grammatical_gender_values_, &gender_state,
		context->GetDiagnostics())) {
		return false;
		}

		float gender_ratio = 0;
		if (!ParseGenderRatio(grammatical_gender_ratio_, &gender_ratio, context->GetDiagnostics())) {
		return false;
		}

		std::random_device rd;
		std::mt19937 gen(rd());
		std::uniform_real_distribution<> distrib(0.0, 1.0);

		for (auto& package : table->packages) {
		for (auto& type : package->types) {
		for (auto& entry : type->entries) {
		bool gender_flag = false;
		if (distrib(gen) < gender_ratio) {
		gender_flag = true;
		}
		std::vector<ResourceConfigValue*> values = entry->FindValuesIf(IsPseudolocalizable);
		for (ResourceConfigValue* value : values) {
		PseudolocalizeIfNeeded(Pseudolocalizer::Method::kAccent, value, &table->string_pool,
		entry.get());
		entry.get(), gender_state, gender_flag);
		PseudolocalizeIfNeeded(Pseudolocalizer::Method::kBidi, value, &table->string_pool,
		entry.get());
		entry.get(), gender_state, gender_flag);
		}
		}
		}

tools/aapt2/compile/PseudolocaleGenerator.h

+13 −2

Original line number	Diff line number	Diff line
		@@ -27,8 +27,19 @@ std::unique_ptr<StyledString> PseudolocalizeStyledString(StyledString* string,
		Pseudolocalizer::Method method,
		android::StringPool* pool);

		struct PseudolocaleGenerator : public IResourceTableConsumer {
		bool Consume(IAaptContext* context, ResourceTable* table) override;
		class PseudolocaleGenerator : public IResourceTableConsumer {
		public:
		explicit PseudolocaleGenerator(std::string grammatical_gender_values,
		std::string grammatical_gender_ratio)
		: grammatical_gender_values_(std::move(grammatical_gender_values)),
		grammatical_gender_ratio_(std::move(grammatical_gender_ratio)) {
		}

		bool Consume(IAaptContext* context, ResourceTable* table);

		private:
		std::string grammatical_gender_values_;
		std::string grammatical_gender_ratio_;
		};

		} // namespace aapt