Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit cf95a580 authored by Adam Lesinski's avatar Adam Lesinski
Browse files

AAPT2: Fix tokenizer to deal with the separator as last char 

When the separator was the last character, the tokenizer would skip
the empty string, which is wrong.

Change-Id: I64ceb4e8d4ac48ee41d40781ab5785a3bc61b24d
parent fc9570e6

tools/aapt2/util/Util.h

+19 −16
Original line number Diff line number Diff line
@@ -229,11 +229,12 @@ public: 
    private:

        friend class Tokenizer<Char>;



        iterator(BasicStringPiece<Char> s, Char sep, BasicStringPiece<Char> tok);

        iterator(BasicStringPiece<Char> s, Char sep, BasicStringPiece<Char> tok, bool end);



        BasicStringPiece<Char> str;

        Char separator;

        BasicStringPiece<Char> token;

        BasicStringPiece<Char> mStr;

        Char mSeparator;

        BasicStringPiece<Char> mToken;

        bool mEnd;

    };



    Tokenizer(BasicStringPiece<Char> str, Char sep);
@@ -252,36 +253,38 @@ inline Tokenizer<Char> tokenize(BasicStringPiece<Char> str, Char sep) { 


template <typename Char>

typename Tokenizer<Char>::iterator& Tokenizer<Char>::iterator::operator++() {

    const Char* start = token.end();

    const Char* end = str.end();

    const Char* start = mToken.end();

    const Char* end = mStr.end();

    if (start == end) {

        token.assign(token.end(), 0);

        mEnd = true;

        mToken.assign(mToken.end(), 0);

        return *this;

    }



    start += 1;

    const Char* current = start;

    while (current != end) {

        if (*current == separator) {

            token.assign(start, current - start);

        if (*current == mSeparator) {

            mToken.assign(start, current - start);

            return *this;

        }

        ++current;

    }

    token.assign(start, end - start);

    mToken.assign(start, end - start);

    return *this;

}



template <typename Char>

inline BasicStringPiece<Char> Tokenizer<Char>::iterator::operator*() {

    return token;

    return mToken;

}



template <typename Char>

inline bool Tokenizer<Char>::iterator::operator==(const iterator& rhs) const {

    // We check equality here a bit differently.

    // We need to know that the addresses are the same.

    return token.begin() == rhs.token.begin() && token.end() == rhs.token.end();

    return mToken.begin() == rhs.mToken.begin() && mToken.end() == rhs.mToken.end() &&

            mEnd == rhs.mEnd;

}



template <typename Char>
@@ -291,8 +294,8 @@ inline bool Tokenizer<Char>::iterator::operator!=(const iterator& rhs) const { 


template <typename Char>

inline Tokenizer<Char>::iterator::iterator(BasicStringPiece<Char> s, Char sep,

                                           BasicStringPiece<Char> tok) :

        str(s), separator(sep), token(tok) {

                                           BasicStringPiece<Char> tok, bool end) :

        mStr(s), mSeparator(sep), mToken(tok), mEnd(end) {

}



template <typename Char>
@@ -307,8 +310,8 @@ inline typename Tokenizer<Char>::iterator Tokenizer<Char>::end() { 


template <typename Char>

inline Tokenizer<Char>::Tokenizer(BasicStringPiece<Char> str, Char sep) :

        mBegin(++iterator(str, sep, BasicStringPiece<Char>(str.begin() - 1, 0))),

        mEnd(str, sep, BasicStringPiece<Char>(str.end(), 0)) {

        mBegin(++iterator(str, sep, BasicStringPiece<Char>(str.begin() - 1, 0), false)),

        mEnd(str, sep, BasicStringPiece<Char>(str.end(), 0), true) {

}



inline uint16_t hostToDevice16(uint16_t value) {

tools/aapt2/util/Util_test.cpp

+9 −0
Original line number Diff line number Diff line
@@ -101,6 +101,15 @@ TEST(UtilTest, TokenizeInput) { 
    ASSERT_EQ(tokenizer.end(), iter);

}



TEST(UtilTest, TokenizeEmptyString) {

    auto tokenizer = util::tokenize(StringPiece16(u""), u'|');

    auto iter = tokenizer.begin();

    ASSERT_NE(tokenizer.end(), iter);

    ASSERT_EQ(StringPiece16(), *iter);

    ++iter;

    ASSERT_EQ(tokenizer.end(), iter);

}



TEST(UtilTest, TokenizeAtEnd) {

    auto tokenizer = util::tokenize(StringPiece16(u"one."), u'.');

    auto iter = tokenizer.begin();