Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 4976e196 authored by Haoyu Zhang's avatar Haoyu Zhang
Browse files

Introduce visible line bounds to CursorAnchorInfo 

Add visible line bounds to CursorAnchorInfo for IME to improve
scribe rich gesture recognization accuracy.

Bug: 215533103
Bug: 239843501
Test: manually tested
Change-Id: I7039b75388ae34d24feb38d18d8da9a91df89504
parent 77689d19

core/api/current.txt

+4 −0
Original line number Diff line number Diff line
@@ -52976,6 +52976,7 @@ package android.view.inputmethod { 
    method public android.graphics.Matrix getMatrix();

    method public int getSelectionEnd();

    method public int getSelectionStart();

    method @NonNull public java.util.List<android.graphics.RectF> getVisibleLineBounds();

    method public void writeToParcel(android.os.Parcel, int);

    field @NonNull public static final android.os.Parcelable.Creator<android.view.inputmethod.CursorAnchorInfo> CREATOR;

    field public static final int FLAG_HAS_INVISIBLE_REGION = 2; // 0x2
@@ -52986,7 +52987,9 @@ package android.view.inputmethod { 
  public static final class CursorAnchorInfo.Builder {

    ctor public CursorAnchorInfo.Builder();

    method public android.view.inputmethod.CursorAnchorInfo.Builder addCharacterBounds(int, float, float, float, float, int);

    method @NonNull public android.view.inputmethod.CursorAnchorInfo.Builder addVisibleLineBounds(float, float, float, float);

    method public android.view.inputmethod.CursorAnchorInfo build();

    method @NonNull public android.view.inputmethod.CursorAnchorInfo.Builder clearVisibleLineBounds();

    method public void reset();

    method public android.view.inputmethod.CursorAnchorInfo.Builder setComposingText(int, CharSequence);

    method @NonNull public android.view.inputmethod.CursorAnchorInfo.Builder setEditorBoundsInfo(@Nullable android.view.inputmethod.EditorBoundsInfo);
@@ -53224,6 +53227,7 @@ package android.view.inputmethod { 
    field public static final int CURSOR_UPDATE_FILTER_CHARACTER_BOUNDS = 8; // 0x8

    field public static final int CURSOR_UPDATE_FILTER_EDITOR_BOUNDS = 4; // 0x4

    field public static final int CURSOR_UPDATE_FILTER_INSERTION_MARKER = 16; // 0x10

    field public static final int CURSOR_UPDATE_FILTER_VISIBLE_LINE_BOUNDS = 32; // 0x20

    field public static final int CURSOR_UPDATE_IMMEDIATE = 1; // 0x1

    field public static final int CURSOR_UPDATE_MONITOR = 2; // 0x2

    field public static final int GET_EXTRACTED_TEXT_MONITOR = 1; // 0x1

core/java/android/view/inputmethod/CursorAnchorInfo.java

+97 −3
Original line number Diff line number Diff line
@@ -27,7 +27,10 @@ import android.text.SpannedString; 
import android.text.TextUtils;

import android.view.inputmethod.SparseRectFArray.SparseRectFArrayBuilder;



import java.util.ArrayList;

import java.util.Arrays;

import java.util.Collections;

import java.util.List;

import java.util.Objects;



/**
@@ -118,6 +121,13 @@ public final class CursorAnchorInfo implements Parcelable { 
    @NonNull

    private final float[] mMatrixValues;



    /**

     * A list of visible line bounds stored in a float array. This array is divided into segment of

     * four where each element in the segment represents left, top, right respectively and bottom

     * of the line bounds.

     */

    private final float[] mVisibleLineBounds;



    /**

     * Flag for {@link #getInsertionMarkerFlags()} and {@link #getCharacterBoundsFlags(int)}: the

     * insertion marker or character bounds have at least one visible region.
@@ -150,6 +160,7 @@ public final class CursorAnchorInfo implements Parcelable { 
        mCharacterBoundsArray = source.readParcelable(SparseRectFArray.class.getClassLoader(), android.view.inputmethod.SparseRectFArray.class);

        mEditorBoundsInfo = source.readTypedObject(EditorBoundsInfo.CREATOR);

        mMatrixValues = source.createFloatArray();

        mVisibleLineBounds = source.createFloatArray();

    }



    /**
@@ -173,6 +184,7 @@ public final class CursorAnchorInfo implements Parcelable { 
        dest.writeParcelable(mCharacterBoundsArray, flags);

        dest.writeTypedObject(mEditorBoundsInfo, flags);

        dest.writeFloatArray(mMatrixValues);

        dest.writeFloatArray(mVisibleLineBounds);

    }



    @Override
@@ -229,6 +241,10 @@ public final class CursorAnchorInfo implements Parcelable { 
            return false;

        }



        if (!Arrays.equals(mVisibleLineBounds, that.mVisibleLineBounds)) {

            return false;

        }



        // Following fields are (partially) covered by hashCode().



        if (mComposingTextStart != that.mComposingTextStart
@@ -262,6 +278,7 @@ public final class CursorAnchorInfo implements Parcelable { 
                + " mInsertionMarkerBottom=" + mInsertionMarkerBottom

                + " mCharacterBoundsArray=" + Objects.toString(mCharacterBoundsArray)

                + " mEditorBoundsInfo=" + mEditorBoundsInfo

                + " mVisibleLineBounds=" + getVisibleLineBounds()

                + " mMatrix=" + Arrays.toString(mMatrixValues)

                + "}";

    }
@@ -270,6 +287,7 @@ public final class CursorAnchorInfo implements Parcelable { 
     * Builder for {@link CursorAnchorInfo}. This class is not designed to be thread-safe.

     */

    public static final class Builder {

        private static final int LINE_BOUNDS_INITIAL_SIZE = 4;

        private int mSelectionStart = -1;

        private int mSelectionEnd = -1;

        private int mComposingTextStart = -1;
@@ -283,6 +301,8 @@ public final class CursorAnchorInfo implements Parcelable { 
        private EditorBoundsInfo mEditorBoundsInfo = null;

        private float[] mMatrixValues = null;

        private boolean mMatrixInitialized = false;

        private float[] mVisibleLineBounds = new float[LINE_BOUNDS_INITIAL_SIZE * 4];

        private int mVisibleLineBoundsCount = 0;



        /**

         * Sets the text range of the selection. Calling this can be skipped if there is no
@@ -395,6 +415,49 @@ public final class CursorAnchorInfo implements Parcelable { 
            return this;

        }



        /**

         * Add the bounds of a visible text line of the current editor.

         *

         * The line bounds should not include the vertical space between lines or the horizontal

         * space before and after a line.

         * It's preferable if the line bounds are added in the logical order, so that IME can

         * process them easily.

         *

         * @param left the left bound of the left-most character in the line

         * @param top the top bound of the top-most character in the line

         * @param right the right bound of the right-most character in the line

         * @param bottom the bottom bound of the bottom-most character in the line

         *

         * @see CursorAnchorInfo#getVisibleLineBounds()

         * @see #clearVisibleLineBounds()

         */

        @NonNull

        public Builder addVisibleLineBounds(float left, float top, float right, float bottom) {

            if (mVisibleLineBounds.length <= mVisibleLineBoundsCount + 4) {

                mVisibleLineBounds =

                        Arrays.copyOf(mVisibleLineBounds, (mVisibleLineBoundsCount + 4) * 2);

            }

            mVisibleLineBounds[mVisibleLineBoundsCount++] = left;

            mVisibleLineBounds[mVisibleLineBoundsCount++] = top;

            mVisibleLineBounds[mVisibleLineBoundsCount++] = right;

            mVisibleLineBounds[mVisibleLineBoundsCount++] = bottom;

            return this;

        }



        /**

         * Clear the visible text line bounds previously added to this {@link Builder}.

         *

         * @see #addVisibleLineBounds(float, float, float, float)

         */

        @NonNull

        public Builder clearVisibleLineBounds() {

            // Since mVisibleLineBounds is copied in build(), we only need to reset

            // mVisibleLineBoundsCount to 0. And mVisibleLineBounds will be reused for better

            // performance.

            mVisibleLineBoundsCount = 0;

            return this;

        }



        /**

         * @return {@link CursorAnchorInfo} using parameters in this {@link Builder}.

         * @throws IllegalArgumentException if one or more positional parameters are specified but
@@ -406,7 +469,10 @@ public final class CursorAnchorInfo implements Parcelable { 
                // parameter is specified.

                final boolean hasCharacterBounds = (mCharacterBoundsArrayBuilder != null

                        && !mCharacterBoundsArrayBuilder.isEmpty());

                final boolean hasVisibleLineBounds = (mVisibleLineBounds != null

                        && mVisibleLineBoundsCount > 0);

                if (hasCharacterBounds

                        || hasVisibleLineBounds

                        || !Float.isNaN(mInsertionMarkerHorizontal)

                        || !Float.isNaN(mInsertionMarkerTop)

                        || !Float.isNaN(mInsertionMarkerBaseline)
@@ -437,6 +503,7 @@ public final class CursorAnchorInfo implements Parcelable { 
                mCharacterBoundsArrayBuilder.reset();

            }

            mEditorBoundsInfo = null;

            clearVisibleLineBounds();

        }

    }
@@ -456,7 +523,8 @@ public final class CursorAnchorInfo implements Parcelable { 
                builder.mComposingTextStart, builder.mComposingText, builder.mInsertionMarkerFlags,

                builder.mInsertionMarkerHorizontal, builder.mInsertionMarkerTop,

                builder.mInsertionMarkerBaseline, builder.mInsertionMarkerBottom,

                characterBoundsArray, builder.mEditorBoundsInfo, matrixValues);

                characterBoundsArray, builder.mEditorBoundsInfo, matrixValues,

                Arrays.copyOf(builder.mVisibleLineBounds, builder.mVisibleLineBoundsCount));

    }



    private CursorAnchorInfo(int selectionStart, int selectionEnd, int composingTextStart,
@@ -465,7 +533,7 @@ public final class CursorAnchorInfo implements Parcelable { 
            float insertionMarkerBaseline, float insertionMarkerBottom,

            @Nullable SparseRectFArray characterBoundsArray,

            @Nullable EditorBoundsInfo editorBoundsInfo,

            @NonNull float[] matrixValues) {

            @NonNull float[] matrixValues, @Nullable float[] visibleLineBounds) {

        mSelectionStart = selectionStart;

        mSelectionEnd = selectionEnd;

        mComposingTextStart = composingTextStart;
@@ -478,6 +546,7 @@ public final class CursorAnchorInfo implements Parcelable { 
        mCharacterBoundsArray = characterBoundsArray;

        mEditorBoundsInfo = editorBoundsInfo;

        mMatrixValues = matrixValues;

        mVisibleLineBounds = visibleLineBounds;



        // To keep hash function simple, we only use some complex objects for hash.

        int hashCode = Objects.hashCode(mComposingText);
@@ -503,7 +572,8 @@ public final class CursorAnchorInfo implements Parcelable { 
                original.mInsertionMarkerFlags, original.mInsertionMarkerHorizontal,

                original.mInsertionMarkerTop, original.mInsertionMarkerBaseline,

                original.mInsertionMarkerBottom, original.mCharacterBoundsArray,

                original.mEditorBoundsInfo, computeMatrixValues(parentMatrix, original));

                original.mEditorBoundsInfo, computeMatrixValues(parentMatrix, original),

                original.mVisibleLineBounds);

    }



    /**
@@ -636,6 +706,30 @@ public final class CursorAnchorInfo implements Parcelable { 
        return mCharacterBoundsArray.getFlags(index, 0);

    }



    /**

     * Returns the list of {@link RectF}s indicating the locations of the visible line bounds in

     * the editor.

     * @return the visible line bounds in the local coordinates as a list of {@link RectF}.

     *

     * @see Builder#addVisibleLineBounds(float, float, float, float)

     */

    @NonNull

    public List<RectF> getVisibleLineBounds() {

        if (mVisibleLineBounds == null) {

            return Collections.emptyList();

        }

        final List<RectF> result = new ArrayList<>(mVisibleLineBounds.length / 4);

        for (int index = 0; index < mVisibleLineBounds.length;) {

            final RectF rectF = new RectF(

                    mVisibleLineBounds[index++],

                    mVisibleLineBounds[index++],

                    mVisibleLineBounds[index++],

                    mVisibleLineBounds[index++]);

            result.add(rectF);

        }

        return result;

    }



    /**

     * Returns {@link EditorBoundsInfo} for the current editor, or {@code null} if IME is not

     * subscribed with {@link InputConnection#CURSOR_UPDATE_FILTER_EDITOR_BOUNDS}

core/java/android/view/inputmethod/InputConnection.java

+24 −4
Original line number Diff line number Diff line
@@ -1070,8 +1070,9 @@ public interface InputConnection { 
     * </p>

     * <p>

     * Note by default all of {@link #CURSOR_UPDATE_FILTER_EDITOR_BOUNDS},

     * {@link #CURSOR_UPDATE_FILTER_CHARACTER_BOUNDS} and

     * {@link #CURSOR_UPDATE_FILTER_INSERTION_MARKER} are included but specifying them can

     * {@link #CURSOR_UPDATE_FILTER_CHARACTER_BOUNDS},

     * {@link #CURSOR_UPDATE_FILTER_VISIBLE_LINE_BOUNDS} and

     * {@link #CURSOR_UPDATE_FILTER_INSERTION_MARKER}, are included but specifying them can

     * filter-out others.

     * It can be CPU intensive to include all, filtering specific info is recommended.

     * </p>
@@ -1085,6 +1086,7 @@ public interface InputConnection { 
     * monitoring, call {@link InputConnection#requestCursorUpdates(int)} again with this flag off.

     * <p>

     * This flag can be used together with filters: {@link #CURSOR_UPDATE_FILTER_CHARACTER_BOUNDS},

     * {@link #CURSOR_UPDATE_FILTER_VISIBLE_LINE_BOUNDS},

     * {@link #CURSOR_UPDATE_FILTER_INSERTION_MARKER} and update flags

     * {@link #CURSOR_UPDATE_IMMEDIATE} and {@link #CURSOR_UPDATE_MONITOR}.

     * </p>
@@ -1099,6 +1101,7 @@ public interface InputConnection { 
     * monitoring, call {@link InputConnection#requestCursorUpdates(int)} again with this flag off.

     * <p>

     * This flag can be combined with other filters: {@link #CURSOR_UPDATE_FILTER_EDITOR_BOUNDS},

     * {@link #CURSOR_UPDATE_FILTER_VISIBLE_LINE_BOUNDS},

     * {@link #CURSOR_UPDATE_FILTER_INSERTION_MARKER} and update flags

     * {@link #CURSOR_UPDATE_IMMEDIATE} and {@link #CURSOR_UPDATE_MONITOR}.

     * </p>
@@ -1114,12 +1117,29 @@ public interface InputConnection { 
     * with this flag off.

     * <p>

     * This flag can be combined with other filters: {@link #CURSOR_UPDATE_FILTER_CHARACTER_BOUNDS},

     * {@link #CURSOR_UPDATE_FILTER_VISIBLE_LINE_BOUNDS},

     * {@link #CURSOR_UPDATE_FILTER_EDITOR_BOUNDS} and update flags {@link #CURSOR_UPDATE_IMMEDIATE}

     * and {@link #CURSOR_UPDATE_MONITOR}.

     * </p>

     */

    int CURSOR_UPDATE_FILTER_INSERTION_MARKER = 1 << 4;



    /**

     * The editor is requested to call

     * {@link InputMethodManager#updateCursorAnchorInfo(android.view.View, CursorAnchorInfo)}

     * with new visible line bounds {@link CursorAnchorInfo#getVisibleLineBounds()} whenever

     * cursor/anchor position is changed, the editor or its parent is scrolled or the line bounds

     * changed due to text updates. To disable monitoring, call

     * {@link InputConnection#requestCursorUpdates(int)} again with this flag off.

     * <p>

     * This flag can be combined with other filters: {@link #CURSOR_UPDATE_FILTER_CHARACTER_BOUNDS},

     * {@link #CURSOR_UPDATE_FILTER_EDITOR_BOUNDS}, {@link #CURSOR_UPDATE_FILTER_INSERTION_MARKER}

     * and update flags {@link #CURSOR_UPDATE_IMMEDIATE}

     * and {@link #CURSOR_UPDATE_MONITOR}.

     * </p>

     */

    int CURSOR_UPDATE_FILTER_VISIBLE_LINE_BOUNDS = 1 << 5;



    /**

     * @hide

     */
@@ -1133,8 +1153,8 @@ public interface InputConnection { 
     */

    @Retention(RetentionPolicy.SOURCE)

    @IntDef(value = {CURSOR_UPDATE_FILTER_EDITOR_BOUNDS, CURSOR_UPDATE_FILTER_CHARACTER_BOUNDS,

            CURSOR_UPDATE_FILTER_INSERTION_MARKER}, flag = true,

            prefix = { "CURSOR_UPDATE_FILTER_" })

            CURSOR_UPDATE_FILTER_INSERTION_MARKER, CURSOR_UPDATE_FILTER_VISIBLE_LINE_BOUNDS},

            flag = true, prefix = { "CURSOR_UPDATE_FILTER_" })

    @interface CursorUpdateFilter{}



    /**

core/java/android/widget/Editor.java

+32 −2
Original line number Diff line number Diff line
@@ -4621,12 +4621,16 @@ public class Editor { 
                    (filter & InputConnection.CURSOR_UPDATE_FILTER_CHARACTER_BOUNDS) != 0;

            boolean includeInsertionMarker =

                    (filter & InputConnection.CURSOR_UPDATE_FILTER_INSERTION_MARKER) != 0;

            boolean includeVisibleLineBounds =

                    (filter & InputConnection.CURSOR_UPDATE_FILTER_VISIBLE_LINE_BOUNDS) != 0;

            boolean includeAll =

                    (!includeEditorBounds && !includeCharacterBounds && !includeInsertionMarker);

                    (!includeEditorBounds && !includeCharacterBounds && !includeInsertionMarker

                    && !includeVisibleLineBounds);



            includeEditorBounds |= includeAll;

            includeCharacterBounds |= includeAll;

            includeInsertionMarker |= includeAll;

            includeVisibleLineBounds |= includeAll;



            final CursorAnchorInfo.Builder builder = mSelectionInfoBuilder;

            builder.reset();
@@ -4655,7 +4659,7 @@ public class Editor { 
                builder.setEditorBoundsInfo(editorBoundsInfo);

            }



            if (includeCharacterBounds || includeInsertionMarker) {

            if (includeCharacterBounds || includeInsertionMarker || includeVisibleLineBounds) {

                final float viewportToContentHorizontalOffset =

                        mTextView.viewportToContentHorizontalOffset();

                final float viewportToContentVerticalOffset =
@@ -4718,6 +4722,32 @@ public class Editor { 
                                insertionMarkerFlags);

                    }

                }



                if (includeVisibleLineBounds) {

                    Rect visibleRect = new Rect();

                    if (mTextView.getLocalVisibleRect(visibleRect)) {

                        final float visibleTop =

                                visibleRect.top + viewportToContentVerticalOffset;

                        final float visibleBottom =

                                visibleRect.bottom + viewportToContentVerticalOffset;

                        final int firstLine =

                                layout.getLineForVertical((int) Math.floor(visibleTop));

                        final int lastLine =

                                layout.getLineForVertical((int) Math.ceil(visibleBottom));



                        for (int line = firstLine; line <= lastLine; ++line) {

                            final float left = layout.getLineLeft(line)

                                    + viewportToContentHorizontalOffset;

                            final float top = layout.getLineTop(line)

                                    + viewportToContentVerticalOffset;

                            final float right = layout.getLineRight(line)

                                    + viewportToContentHorizontalOffset;

                            final float bottom = layout.getLineBottom(line)

                                    + viewportToContentVerticalOffset;

                            builder.addVisibleLineBounds(left, top, right, bottom);

                        }

                    }

                }

            }



            imm.updateCursorAnchorInfo(mTextView, builder.build());