Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 6576dba0 authored by cketti's avatar cketti
Browse files

Remove unused `TagTokenizer`

parent 11b989ec
Loading
Loading
Loading
Loading
+0 −113
Original line number Original line Diff line number Diff line
package com.tokenautocomplete;

import android.os.Parcel;
import android.os.Parcelable;
import androidx.annotation.NonNull;

import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;

@SuppressWarnings("unused")
public class TagTokenizer implements Tokenizer {

    private ArrayList<Character> tagPrefixes;

    TagTokenizer() {
        this(Arrays.asList('@', '#'));
    }

    public TagTokenizer(List<Character> tagPrefixes){
        super();
        this.tagPrefixes = new ArrayList<>(tagPrefixes);
    }

    @SuppressWarnings("WeakerAccess")
    protected boolean isTokenTerminator(char character) {
        //Allow letters, numbers and underscores
        return !Character.isLetterOrDigit(character) && character != '_';
    }

    @Override
    public boolean containsTokenTerminator(CharSequence charSequence) {
        for (int i = 0; i < charSequence.length(); ++i) {
            if (isTokenTerminator(charSequence.charAt(i))) {
                return true;
            }
        }

        return false;
    }

    @Override
    @NonNull
    public List<Range> findTokenRanges(CharSequence charSequence, int start, int end) {
        ArrayList<Range>result = new ArrayList<>();

        if (start == end) {
            //Can't have a 0 length token
            return result;
        }

        int tokenStart = Integer.MAX_VALUE;

        for (int cursor = start; cursor < end; ++cursor) {
            char character = charSequence.charAt(cursor);

            //Either this is a terminator, or we contain some content and are at the end of input
            if (isTokenTerminator(character)) {
                //Is there some token content? Might just be two terminators in a row
                if (cursor - 1 > tokenStart) {
                    result.add(new Range(tokenStart, cursor));
                }

                //mark that we don't have a candidate token start any more
                tokenStart = Integer.MAX_VALUE;
            }

            //Set tokenStart when we hit a tag prefix
            if (tagPrefixes.contains(character)) {
                tokenStart = cursor;
            }
        }

        if (end > tokenStart) {
            //There was unterminated text after a start of token
            result.add(new Range(tokenStart, end));
        }

        return result;
    }

    @Override
    @NonNull
    public CharSequence wrapTokenValue(CharSequence text) {
        return text;
    }

    public static final Parcelable.Creator<TagTokenizer> CREATOR = new Parcelable.Creator<TagTokenizer>() {
        @SuppressWarnings("unchecked")
        public TagTokenizer createFromParcel(Parcel in) {
            return new TagTokenizer(in);
        }

        public TagTokenizer[] newArray(int size) {
            return new TagTokenizer[size];
        }
    };

    @Override
    public int describeContents() {
        return 0;
    }

    @SuppressWarnings({"WeakerAccess", "unchecked"})
    TagTokenizer(Parcel in) {
        this(in.readArrayList(Character.class.getClassLoader()));
    }

    @Override
    public void writeToParcel(Parcel parcel, int i) {
        parcel.writeList(tagPrefixes);
    }
}
+0 −74
Original line number Original line Diff line number Diff line
package com.tokenautocomplete;

import org.junit.Before;
import org.junit.Test;

import java.util.Arrays;
import java.util.Collections;
import java.util.List;

import static org.junit.Assert.assertEquals;

public class TagTokenizerTest {

    private TagTokenizer tokenizer;

    @Before
    public void setup() {
        tokenizer = new TagTokenizer();
    }

    @Test
    public void testBasicTagDetection() {
        String test = "@bears #tokens";
        List<Range> ranges = tokenizer.findTokenRanges(test, 0, test.length());
        assertEquals(Arrays.asList(new Range(0, 6), new Range(7, 14)), ranges);
        assertEquals("@bears", test.substring(ranges.get(0).start, ranges.get(0).end));
        assertEquals("#tokens", test.substring(ranges.get(1).start, ranges.get(1).end));
    }

    @Test
    public void testSequentialTagDetection() {
        String test = "@@bears#tokens#";
        assertEquals(Arrays.asList(new Range(1,7), new Range(7,14), new Range(14, 15)),
                tokenizer.findTokenRanges(test, 0, test.length()));
    }

    @Test
    public void testNonTokenInput() {
        String test = "This is some input with @names and #hash #tags inside";
        assertEquals(Arrays.asList(new Range(24,30), new Range(35,40), new Range(41,46)),
                tokenizer.findTokenRanges(test, 0, test.length()));
    }

    @Test
    public void testMissingTokenContentInput() {
        String test = "@token       @ asdm      @asjdfhajks      sdfasdf";
        assertEquals(Arrays.asList(new Range(0,6), new Range(25, 36)),
                tokenizer.findTokenRanges(test, 0, test.length()));
    }

    @Test
    public void allowsOneCharacterCandidateRangeMatches() {
        String text = "#";

        List<Range> ranges = tokenizer.findTokenRanges(text, 0, text.length());
        assertEquals(Collections.singletonList(new Range(0,1)), ranges);
    }

    @Test
    public void allowsOneCharacterCandidateRangeMatchesWithWhitespace() {
        String text = " #";

        List<Range> ranges = tokenizer.findTokenRanges(text, 0, text.length());
        assertEquals(Collections.singletonList(new Range(1,2)), ranges);
    }

    @Test
    public void matchesSingleLetterTokens() {
        String text = "#t#r #a##b";

        List<Range> ranges = tokenizer.findTokenRanges(text, 0, text.length());
        assertEquals(Arrays.asList(new Range(0, 2), new Range(2,4), new Range(5,7), new Range(8,10)), ranges);
    }
}