Merge "AAPT2: Fix styled string whitespace processing"

    <!-- Text message in the factory reset warning dialog. This says that the the device admin app

         is missing or corrupted. As a result the device will be erased. [CHAR LIMIT=NONE]-->

    <string name="factory_reset_message">The admin app can\'t be used. Your device will now be

        erased.\n\nIf you have questions, contact your organization's admin.</string>

        erased.\n\nIf you have questions, contact your organization\'s admin.</string>

    <!-- A toast message displayed when printing is attempted but disabled by policy. -->

    <string name="printing_disabled_by">Printing disabled by <xliff:g id="owner_app">%s</xliff:g>.</string>

    <string name="capability_title_canCaptureFingerprintGestures">Fingerprint gestures</string>

    <!-- Description for the capability of an accessibility service to perform gestures. -->

    <string name="capability_desc_canCaptureFingerprintGestures">Can capture gestures performed on

        the device's fingerprint sensor.</string>

        the device\'s fingerprint sensor.</string>

    <!--  Permissions -->

    <!-- Notification title when data usage has exceeded warning threshold. [CHAR LIMIT=50] -->

    <string name="data_usage_warning_title">Data warning</string>

    <!-- Notification body when data usage has exceeded warning threshold. [CHAR LIMIT=32] -->

    <string name="data_usage_warning_body">You've used <xliff:g id="app" example="3.8GB">%s</xliff:g> of data</string>

    <string name="data_usage_warning_body">You\'ve used <xliff:g id="app" example="3.8GB">%s</xliff:g> of data</string>

    <!-- Notification title when mobile data usage has exceeded limit threshold, and has been disabled. [CHAR LIMIT=50] -->

    <string name="data_usage_mobile_limit_title">Mobile data limit reached</string>

    <!-- Notification title when Wi-Fi data usage has exceeded limit threshold. [CHAR LIMIT=32] -->

    <string name="data_usage_wifi_limit_snoozed_title">Over your Wi-Fi data limit</string>

    <!-- Notification body when data usage has exceeded limit threshold. -->

    <string name="data_usage_limit_snoozed_body">You've gone <xliff:g id="size" example="3.8GB">%s</xliff:g> over your set limit</string>

    <string name="data_usage_limit_snoozed_body">You\'ve gone <xliff:g id="size" example="3.8GB">%s</xliff:g> over your set limit</string>

    <!-- Notification title when background data usage is limited. [CHAR LIMIT=32] -->

    <string name="data_usage_restricted_title">Background data restricted</string>

#include "ResourceUtils.h"

#include "ResourceValues.h"

#include "ValueVisitor.h"

#include "text/Utf8Iterator.h"

#include "util/ImmutableMap.h"

#include "util/Maybe.h"

#include "util/Util.h"

#include "xml/XmlPullParser.h"

using ::aapt::ResourceUtils::StringBuilder;

using ::aapt::text::Utf8Iterator;

using ::android::StringPiece;

namespace aapt {

      config_(config),

      options_(options) {}

/**

 * Build a string from XML that converts nested elements into Span objects.

 */

// Base class Node for representing the various Spans and UntranslatableSections of an XML string.

// This will be used to traverse and flatten the XML string into a single std::string, with all

// Span and Untranslatable data maintained in parallel, as indices into the string.

class Node {

 public:

  virtual ~Node() = default;

  // Adds the given child node to this parent node's set of child nodes, moving ownership to the

  // parent node as well.

  // Returns a pointer to the child node that was added as a convenience.

  template <typename T>

  T* AddChild(std::unique_ptr<T> node) {

    T* raw_ptr = node.get();

    children.push_back(std::move(node));

    return raw_ptr;

  }

  virtual void Build(StringBuilder* builder) const {

    for (const auto& child : children) {

      child->Build(builder);

    }

  }

  std::vector<std::unique_ptr<Node>> children;

};

// A chunk of text in the XML string. This lives between other tags, such as XLIFF tags and Spans.

class SegmentNode : public Node {

 public:

  std::string data;

  void Build(StringBuilder* builder) const override {

    builder->AppendText(data);

  }

};

// A tag that will be encoded into the final flattened string. Tags like <b> or <i>.

class SpanNode : public Node {

 public:

  std::string name;

  void Build(StringBuilder* builder) const override {

    StringBuilder::SpanHandle span_handle = builder->StartSpan(name);

    Node::Build(builder);

    builder->EndSpan(span_handle);

  }

};

// An XLIFF 'g' tag, which marks a section of the string as untranslatable.

class UntranslatableNode : public Node {

 public:

  void Build(StringBuilder* builder) const override {

    StringBuilder::UntranslatableHandle handle = builder->StartUntranslatable();

    Node::Build(builder);

    builder->EndUntranslatable(handle);

  }

};

// Build a string from XML that converts nested elements into Span objects.

bool ResourceParser::FlattenXmlSubtree(

    xml::XmlPullParser* parser, std::string* out_raw_string, StyleString* out_style_string,

    std::vector<UntranslatableSection>* out_untranslatable_sections) {

  // Keeps track of formatting tags (<b>, <i>) and the range of characters for which they apply.

  // The stack elements refer to the indices in out_style_string->spans.

  // By first adding to the out_style_string->spans vector, and then using the stack to refer

  // to this vector, the original order of tags is preserved in cases such as <b><i>hello</b></i>.

  std::vector<size_t> span_stack;

  // Clear the output variables.

  out_raw_string->clear();

  out_style_string->spans.clear();

  out_untranslatable_sections->clear();

  // The StringBuilder will concatenate the various segments of text which are initially

  // separated by tags. It also handles unicode escape codes and quotations.

  util::StringBuilder builder;

  std::string raw_string;

  std::string current_text;

  // The first occurrence of a <xliff:g> tag. Nested <xliff:g> tags are illegal.

  Maybe<size_t> untranslatable_start_depth;

  Node root;

  std::vector<Node*> node_stack;

  node_stack.push_back(&root);

  bool saw_span_node = false;

  SegmentNode* first_segment = nullptr;

  SegmentNode* last_segment = nullptr;

  size_t depth = 1;

  while (xml::XmlPullParser::IsGoodEvent(parser->Next())) {

  while (depth > 0 && xml::XmlPullParser::IsGoodEvent(parser->Next())) {

    const xml::XmlPullParser::Event event = parser->event();

    if (event == xml::XmlPullParser::Event::kStartElement) {

    // First take care of any SegmentNodes that should be created.

    if (event == xml::XmlPullParser::Event::kStartElement ||

        event == xml::XmlPullParser::Event::kEndElement) {

      if (!current_text.empty()) {

        std::unique_ptr<SegmentNode> segment_node = util::make_unique<SegmentNode>();

        segment_node->data = std::move(current_text);

        last_segment = node_stack.back()->AddChild(std::move(segment_node));

        if (first_segment == nullptr) {

          first_segment = last_segment;

        }

        current_text = {};

      }

    }

    switch (event) {

      case xml::XmlPullParser::Event::kText: {

        current_text += parser->text();

        raw_string += parser->text();

      } break;

      case xml::XmlPullParser::Event::kStartElement: {

        if (parser->element_namespace().empty()) {

          // This is an HTML tag which we encode as a span. Add it to the span stack.

        std::string span_name = parser->element_name();

          std::unique_ptr<SpanNode> span_node = util::make_unique<SpanNode>();

          span_node->name = parser->element_name();

          const auto end_attr_iter = parser->end_attributes();

        for (auto attr_iter = parser->begin_attributes(); attr_iter != end_attr_iter; ++attr_iter) {

          span_name += ";";

          span_name += attr_iter->name;

          span_name += "=";

          span_name += attr_iter->value;

          for (auto attr_iter = parser->begin_attributes(); attr_iter != end_attr_iter;

               ++attr_iter) {

            span_node->name += ";";

            span_node->name += attr_iter->name;

            span_node->name += "=";

            span_node->name += attr_iter->value;

          }

        // Make sure the string is representable in our binary format.

        if (builder.Utf16Len() > std::numeric_limits<uint32_t>::max()) {

          diag_->Error(DiagMessage(source_.WithLine(parser->line_number()))

                       << "style string '" << builder.ToString() << "' is too long");

          return false;

        }

        out_style_string->spans.push_back(

            Span{std::move(span_name), static_cast<uint32_t>(builder.Utf16Len())});

        span_stack.push_back(out_style_string->spans.size() - 1);

          node_stack.push_back(node_stack.back()->AddChild(std::move(span_node)));

          saw_span_node = true;

        } else if (parser->element_namespace() == sXliffNamespaceUri) {

          // This is an XLIFF tag, which is not encoded as a span.

          if (parser->element_name() == "g") {

            // Check that an 'untranslatable' tag is not already being processed. Nested

            // <xliff:g> tags are illegal.

            if (untranslatable_start_depth) {

            // We've already encountered an <xliff:g> tag, and nested <xliff:g> tags are illegal.

              diag_->Error(DiagMessage(source_.WithLine(parser->line_number()))

                           << "illegal nested XLIFF 'g' tag");

              return false;

            } else {

            // Mark the start of an untranslatable section. Use UTF8 indices/lengths.

              // Mark the beginning of an 'untranslatable' section.

              untranslatable_start_depth = depth;

            const size_t current_idx = builder.ToString().size();

            out_untranslatable_sections->push_back(UntranslatableSection{current_idx, current_idx});

              node_stack.push_back(

                  node_stack.back()->AddChild(util::make_unique<UntranslatableNode>()));

            }

          } else {

            // Ignore unknown XLIFF tags, but don't warn.

            node_stack.push_back(node_stack.back()->AddChild(util::make_unique<Node>()));

          }

        // Ignore other xliff tags, they get handled by other tools.

        } else {

          // Besides XLIFF, any other namespaced tag is unsupported and ignored.

          diag_->Warn(DiagMessage(source_.WithLine(parser->line_number()))

                      << "ignoring element '" << parser->element_name()

                      << "' with unknown namespace '" << parser->element_namespace() << "'");

          node_stack.push_back(node_stack.back()->AddChild(util::make_unique<Node>()));

        }

        // Enter one level inside the element.

        depth++;

    } else if (event == xml::XmlPullParser::Event::kText) {

      // Record both the raw text and append to the builder to deal with escape sequences

      // and quotations.

      out_raw_string->append(parser->text());

      builder.Append(parser->text());

    } else if (event == xml::XmlPullParser::Event::kEndElement) {

      } break;

      case xml::XmlPullParser::Event::kEndElement: {

        // Return one level from within the element.

        depth--;

        if (depth == 0) {

          break;

        }

      if (parser->element_namespace().empty()) {

        // This is an HTML tag which we encode as a span. Update the span

        // stack and pop the top entry.

        Span& top_span = out_style_string->spans[span_stack.back()];

        top_span.last_char = builder.Utf16Len() - 1;

        span_stack.pop_back();

      } else if (untranslatable_start_depth == make_value(depth)) {

        // This is the end of an untranslatable section. Use UTF8 indices/lengths.

        UntranslatableSection& untranslatable_section = out_untranslatable_sections->back();

        untranslatable_section.end = builder.ToString().size();

        node_stack.pop_back();

        if (untranslatable_start_depth == make_value(depth)) {

          // This is the end of an untranslatable section.

          untranslatable_start_depth = {};

        }

    } else if (event == xml::XmlPullParser::Event::kComment) {

      // Ignore.

    } else {

      LOG(FATAL) << "unhandled XML event";

      } break;

      default:

        // ignore.

        break;

    }

  }

  // Sanity check to make sure we processed all the nodes.

  CHECK(node_stack.size() == 1u);

  CHECK(node_stack.back() == &root);

  if (!saw_span_node) {

    // If there were no spans, we must treat this string a little differently (according to AAPT).

    // Find and strip the leading whitespace from the first segment, and the trailing whitespace

    // from the last segment.

    if (first_segment != nullptr) {

      // Trim leading whitespace.

      StringPiece trimmed = util::TrimLeadingWhitespace(first_segment->data);

      if (trimmed.size() != first_segment->data.size()) {

        first_segment->data = trimmed.to_string();

      }

    }

    if (last_segment != nullptr) {

      // Trim trailing whitespace.

      StringPiece trimmed = util::TrimTrailingWhitespace(last_segment->data);

      if (trimmed.size() != last_segment->data.size()) {

        last_segment->data = trimmed.to_string();

      }

    }

  }

  // Have the XML structure flatten itself into the StringBuilder. The StringBuilder will take

  // care of recording the correctly adjusted Spans and UntranslatableSections.

  StringBuilder builder;

  root.Build(&builder);

  if (!builder) {

    diag_->Error(DiagMessage(source_.WithLine(parser->line_number())) << builder.GetError());

    return false;

  }

  CHECK(span_stack.empty()) << "spans haven't been fully processed";

  out_style_string->str = builder.ToString();

  ResourceUtils::FlattenedXmlString flattened_string = builder.GetFlattenedString();

  *out_raw_string = std::move(raw_string);

  *out_untranslatable_sections = std::move(flattened_string.untranslatable_sections);

  out_style_string->str = std::move(flattened_string.text);

  out_style_string->spans = std::move(flattened_string.spans);

  return true;

}

  ASSERT_THAT(str, NotNull());

  EXPECT_THAT(*str, StrValueEq("  hey there "));

  EXPECT_THAT(str->untranslatable_sections, IsEmpty());

  ASSERT_TRUE(TestParse(R"(<string name="bar">Isn\'t it cool?</string>)"));

  str = test::GetValue<String>(&table_, "string/bar");

  ASSERT_THAT(str, NotNull());

  EXPECT_THAT(*str, StrValueEq("Isn't it cool?"));

  ASSERT_TRUE(TestParse(R"(<string name="baz">"Isn't it cool?"</string>)"));

  str = test::GetValue<String>(&table_, "string/baz");

  ASSERT_THAT(str, NotNull());

  EXPECT_THAT(*str, StrValueEq("Isn't it cool?"));

}

TEST_F(ResourceParserTest, ParseEscapedString) {

  StyledString* str = test::GetValue<StyledString>(&table_, "string/foo");

  ASSERT_THAT(str, NotNull());

  EXPECT_THAT(str->value->value, Eq("This is my aunt\u2019s fickle string"));

  EXPECT_THAT(str->value->value, StrEq("This is my aunt\u2019s fickle string"));

  EXPECT_THAT(str->value->spans, SizeIs(2));

  EXPECT_THAT(str->untranslatable_sections, IsEmpty());

  EXPECT_THAT(*str->value->spans[0].name, Eq("b"));

  EXPECT_THAT(str->value->spans[0].first_char, Eq(17u));

  EXPECT_THAT(*str->value->spans[0].name, StrEq("b"));

  EXPECT_THAT(str->value->spans[0].first_char, Eq(18u));

  EXPECT_THAT(str->value->spans[0].last_char, Eq(30u));

  EXPECT_THAT(*str->value->spans[1].name, Eq("small"));

  EXPECT_THAT(str->value->spans[1].first_char, Eq(24u));

  EXPECT_THAT(*str->value->spans[1].name, StrEq("small"));

  EXPECT_THAT(str->value->spans[1].first_char, Eq(25u));

  EXPECT_THAT(str->value->spans[1].last_char, Eq(30u));

}

  String* str = test::GetValue<String>(&table_, "string/foo");

  ASSERT_THAT(str, NotNull());

  EXPECT_THAT(*str->value, Eq("This is what I think"));

  EXPECT_THAT(*str->value, StrEq("This is what I think"));

  EXPECT_THAT(str->untranslatable_sections, IsEmpty());

  ASSERT_TRUE(TestParse(R"(<string name="foo2">"  This is what  I think  "</string>)"));

  EXPECT_THAT(*str, StrValueEq("  This is what  I think  "));

}

TEST_F(ResourceParserTest, ParseStyledStringWithWhitespace) {

  std::string input = R"(<string name="foo">  <b> My <i> favorite</i> string </b>  </string>)";

  ASSERT_TRUE(TestParse(input));

  StyledString* str = test::GetValue<StyledString>(&table_, "string/foo");

  ASSERT_THAT(str, NotNull());

  EXPECT_THAT(str->value->value, StrEq("  My  favorite string  "));

  EXPECT_THAT(str->untranslatable_sections, IsEmpty());

  ASSERT_THAT(str->value->spans, SizeIs(2u));

  EXPECT_THAT(*str->value->spans[0].name, StrEq("b"));

  EXPECT_THAT(str->value->spans[0].first_char, Eq(1u));

  EXPECT_THAT(str->value->spans[0].last_char, Eq(21u));

  EXPECT_THAT(*str->value->spans[1].name, StrEq("i"));

  EXPECT_THAT(str->value->spans[1].first_char, Eq(5u));

  EXPECT_THAT(str->value->spans[1].last_char, Eq(13u));

}

TEST_F(ResourceParserTest, IgnoreXliffTagsOtherThanG) {

  std::string input = R"(

      <string name="foo" xmlns:xliff="urn:oasis:names:tc:xliff:document:1.2">

  String* str = test::GetValue<String>(&table_, "string/foo");

  ASSERT_THAT(str, NotNull());

  EXPECT_THAT(*str, StrValueEq("There are %1$d apples"));

  ASSERT_THAT(str->untranslatable_sections, SizeIs(1));

  // We expect indices and lengths that span to include the whitespace

  // before %1$d. This is due to how the StringBuilder withholds whitespace unless

  // needed (to deal with line breaks, etc.).

  EXPECT_THAT(str->untranslatable_sections[0].start, Eq(9u));

  ASSERT_THAT(str->untranslatable_sections, SizeIs(1));

  EXPECT_THAT(str->untranslatable_sections[0].start, Eq(10u));

  EXPECT_THAT(str->untranslatable_sections[0].end, Eq(14u));

}

  StyledString* str = test::GetValue<StyledString>(&table_, "string/foo");

  ASSERT_THAT(str, NotNull());

  EXPECT_THAT(str->value->value, Eq(" There are %1$d apples"));

  ASSERT_THAT(str->untranslatable_sections, SizeIs(1));

  EXPECT_THAT(str->untranslatable_sections[0].start, Eq(11u));

  EXPECT_THAT(str->untranslatable_sections[0].end, Eq(15u));

  // We expect indices and lengths that span to include the whitespace

  // before %1$d. This is due to how the StringBuilder withholds whitespace unless

  // needed (to deal with line breaks, etc.).

  EXPECT_THAT(str->untranslatable_sections[0].start, Eq(9u));

  EXPECT_THAT(str->untranslatable_sections[0].end, Eq(14u));

  ASSERT_THAT(str->value->spans, SizeIs(1u));

  EXPECT_THAT(*str->value->spans[0].name, StrEq("b"));

  EXPECT_THAT(str->value->spans[0].first_char, Eq(11u));

  EXPECT_THAT(str->value->spans[0].last_char, Eq(14u));

}

TEST_F(ResourceParserTest, ParseNull) {

#include <sstream>

#include "android-base/stringprintf.h"

#include "androidfw/ResourceTypes.h"

#include "androidfw/ResourceUtils.h"

#include "NameMangler.h"

#include "SdkConstants.h"

#include "format/binary/ResourceTypeExtensions.h"

#include "text/Unicode.h"

#include "text/Utf8Iterator.h"

#include "util/Files.h"

#include "util/Util.h"

using ::aapt::text::IsWhitespace;

using ::aapt::text::Utf8Iterator;

using ::android::StringPiece;

using ::android::StringPiece16;

using ::android::base::StringPrintf;

namespace aapt {

namespace ResourceUtils {

  return util::make_unique<BinaryPrimitive>(res_value);

}

// Converts the codepoint to UTF-8 and appends it to the string.

static bool AppendCodepointToUtf8String(char32_t codepoint, std::string* output) {

  ssize_t len = utf32_to_utf8_length(&codepoint, 1);

  if (len < 0) {

    return false;

  }

  const size_t start_append_pos = output->size();

  // Make room for the next character.

  output->resize(output->size() + len);

  char* dst = &*(output->begin() + start_append_pos);

  utf32_to_utf8(&codepoint, 1, dst, len + 1);

  return true;

}

// Reads up to 4 UTF-8 characters that represent a Unicode escape sequence, and appends the

// Unicode codepoint represented by the escape sequence to the string.

static bool AppendUnicodeEscapeSequence(Utf8Iterator* iter, std::string* output) {

  char32_t code = 0;

  for (size_t i = 0; i < 4 && iter->HasNext(); i++) {

    char32_t codepoint = iter->Next();

    char32_t a;

    if (codepoint >= U'0' && codepoint <= U'9') {

      a = codepoint - U'0';

    } else if (codepoint >= U'a' && codepoint <= U'f') {

      a = codepoint - U'a' + 10;

    } else if (codepoint >= U'A' && codepoint <= U'F') {

      a = codepoint - U'A' + 10;

    } else {

      return {};

    }

    code = (code << 4) | a;

  }

  return AppendCodepointToUtf8String(code, output);

}

StringBuilder::StringBuilder(bool preserve_spaces)

    : preserve_spaces_(preserve_spaces), quote_(preserve_spaces) {

}

StringBuilder& StringBuilder::AppendText(const std::string& text) {

  if (!error_.empty()) {

    return *this;

  }

  const size_t previous_len = xml_string_.text.size();

  Utf8Iterator iter(text);

  while (iter.HasNext()) {

    char32_t codepoint = iter.Next();

    if (!quote_ && text::IsWhitespace(codepoint)) {

      if (!last_codepoint_was_space_) {

        // Emit a space if it's the first.

        xml_string_.text += ' ';

        last_codepoint_was_space_ = true;

      }

      // Keep eating spaces.

      continue;

    }

    // This is not a space.

    last_codepoint_was_space_ = false;

    if (codepoint == U'\\') {

      if (iter.HasNext()) {

        codepoint = iter.Next();

        switch (codepoint) {

          case U't':

            xml_string_.text += '\t';

            break;

          case U'n':

            xml_string_.text += '\n';

            break;

          case U'#':

          case U'@':

          case U'?':

          case U'"':

          case U'\'':

          case U'\\':

            xml_string_.text += static_cast<char>(codepoint);

            break;

          case U'u':

            if (!AppendUnicodeEscapeSequence(&iter, &xml_string_.text)) {

              error_ =

                  StringPrintf("invalid unicode escape sequence in string\n\"%s\"", text.c_str());

              return *this;

            }

            break;

          default:

            // Ignore the escape character and just include the codepoint.

            AppendCodepointToUtf8String(codepoint, &xml_string_.text);

            break;

        }

      }

    } else if (!preserve_spaces_ && codepoint == U'"') {

      // Only toggle the quote state when we are not preserving spaces.

      quote_ = !quote_;

    } else if (!quote_ && codepoint == U'\'') {

      // This should be escaped.

      error_ = StringPrintf("unescaped apostrophe in string\n\"%s\"", text.c_str());

      return *this;

    } else {

      AppendCodepointToUtf8String(codepoint, &xml_string_.text);

    }

  }

  // Accumulate the added string's UTF-16 length.

  const uint8_t* utf8_data = reinterpret_cast<const uint8_t*>(xml_string_.text.c_str());

  const size_t utf8_length = xml_string_.text.size();

  ssize_t len = utf8_to_utf16_length(utf8_data + previous_len, utf8_length - previous_len);

  if (len < 0) {

    error_ = StringPrintf("invalid unicode code point in string\n\"%s\"", utf8_data + previous_len);

    return *this;

  }

  utf16_len_ += static_cast<uint32_t>(len);

  return *this;

}

StringBuilder::SpanHandle StringBuilder::StartSpan(const std::string& name) {

  if (!error_.empty()) {

    return 0u;

  }

  // When we start a span, all state associated with whitespace truncation and quotation is ended.

  ResetTextState();

  Span span;

  span.name = name;

  span.first_char = span.last_char = utf16_len_;

  xml_string_.spans.push_back(std::move(span));

  return xml_string_.spans.size() - 1;

}

void StringBuilder::EndSpan(SpanHandle handle) {