Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 43aae6ff authored by Haoyu Zhang's avatar Haoyu Zhang
Browse files

Introduce TextBoundsInfo 

This is an API for IME to request text bounds information from editor to
perform handwriting rich gestures. It's useful for IMEs to create custom
handwriting gesture that's not supported the HandwritingGesture. And
it can also be used to provide default rich gesture implementation for
handwriting gesture compatibility.

Bug: 239843501
Test: atest TextBoundsInfoTest
Change-Id: Ia7522371976a4e1fcdff4cc0d244ea427ad3d7c9
parent 900dd54f

core/api/current.txt

+53 −0
Original line number Diff line number Diff line
@@ -45561,6 +45561,14 @@ package android.text { 
    field public static final int DONE = -1; // 0xffffffff

  }













  public static class SegmentFinder.DefaultSegmentFinder extends android.text.SegmentFinder {













    ctor public SegmentFinder.DefaultSegmentFinder(@NonNull int[]);













    method public int nextEndBoundary(@IntRange(from=0) int);













    method public int nextStartBoundary(@IntRange(from=0) int);













    method public int previousEndBoundary(@IntRange(from=0) int);













    method public int previousStartBoundary(@IntRange(from=0) int);













  }



























  public class Selection {















    method public static boolean extendDown(android.text.Spannable, android.text.Layout);















    method public static boolean extendLeft(android.text.Spannable, android.text.Layout);










@@ -53568,6 +53576,7 @@ package android.view.inputmethod {













    method public boolean reportFullscreenMode(boolean);















    method public boolean requestCursorUpdates(int);















    method public default boolean requestCursorUpdates(int, int);













    method public default void requestTextBoundsInfo(@NonNull android.graphics.RectF, @NonNull java.util.concurrent.Executor, @NonNull java.util.function.Consumer<android.view.inputmethod.TextBoundsInfoResult>);















    method public boolean sendKeyEvent(android.view.KeyEvent);















    method public boolean setComposingRegion(int, int);















    method public default boolean setComposingRegion(int, int, @Nullable android.view.inputmethod.TextAttribute);










@@ -53897,6 +53906,50 @@ package android.view.inputmethod {













    method @NonNull public android.view.inputmethod.TextAttribute.Builder setTextConversionSuggestions(@NonNull java.util.List<java.lang.String>);















  }



























  public final class TextBoundsInfo implements android.os.Parcelable {













    method public int describeContents();













    method @IntRange(from=0, to=125) public int getCharacterBidiLevel(int);













    method @NonNull public android.graphics.RectF getCharacterBounds(int);













    method public int getCharacterFlags(int);













    method public int getEnd();













    method @NonNull public android.text.SegmentFinder getGraphemeSegmentFinder();













    method @NonNull public android.text.SegmentFinder getLineSegmentFinder();













    method @NonNull public android.graphics.Matrix getMatrix();













    method public int getStart();













    method @NonNull public android.text.SegmentFinder getWordSegmentFinder();













    method public void writeToParcel(@NonNull android.os.Parcel, int);













    field @NonNull public static final android.os.Parcelable.Creator<android.view.inputmethod.TextBoundsInfo> CREATOR;













    field public static final int FLAG_CHARACTER_LINEFEED = 2; // 0x2













    field public static final int FLAG_CHARACTER_PUNCTUATION = 4; // 0x4













    field public static final int FLAG_CHARACTER_WHITESPACE = 1; // 0x1













    field public static final int FLAG_LINE_IS_RTL = 8; // 0x8













  }

























  public static final class TextBoundsInfo.Builder {













    ctor public TextBoundsInfo.Builder();













    method @NonNull public android.view.inputmethod.TextBoundsInfo build();













    method @NonNull public android.view.inputmethod.TextBoundsInfo.Builder clear();













    method @NonNull public android.view.inputmethod.TextBoundsInfo.Builder setCharacterBidiLevel(@NonNull int[]);













    method @NonNull public android.view.inputmethod.TextBoundsInfo.Builder setCharacterBounds(@NonNull float[]);













    method @NonNull public android.view.inputmethod.TextBoundsInfo.Builder setCharacterFlags(@NonNull int[]);













    method @NonNull public android.view.inputmethod.TextBoundsInfo.Builder setGraphemeSegmentFinder(@NonNull android.text.SegmentFinder);













    method @NonNull public android.view.inputmethod.TextBoundsInfo.Builder setLineSegmentFinder(@NonNull android.text.SegmentFinder);













    method @NonNull public android.view.inputmethod.TextBoundsInfo.Builder setMatrix(@NonNull android.graphics.Matrix);













    method @NonNull public android.view.inputmethod.TextBoundsInfo.Builder setStartAndEnd(@IntRange(from=0) int, @IntRange(from=0) int);













    method @NonNull public android.view.inputmethod.TextBoundsInfo.Builder setWordSegmentFinder(@NonNull android.text.SegmentFinder);













  }

























  public final class TextBoundsInfoResult {













    ctor public TextBoundsInfoResult(int);













    ctor public TextBoundsInfoResult(int, @NonNull android.view.inputmethod.TextBoundsInfo);













    method public int getResultCode();













    method @Nullable public android.view.inputmethod.TextBoundsInfo getTextBoundsInfo();













    field public static final int CODE_CANCELLED = 3; // 0x3













    field public static final int CODE_FAILED = 2; // 0x2













    field public static final int CODE_SUCCESS = 1; // 0x1













    field public static final int CODE_UNSUPPORTED = 0; // 0x0













  }



























  public final class TextSnapshot {















    ctor public TextSnapshot(@NonNull android.view.inputmethod.SurroundingText, @IntRange(from=0xffffffff) int, @IntRange(from=0xffffffff) int, int);















    method @IntRange(from=0xffffffff) public int getCompositionEnd();

























core/java/android/inputmethodservice/IRemoteInputConnectionInvoker.java



+66
−0




























Original line number
Diff line number
Diff line








@@ -16,10 +16,13 @@





























package android.inputmethodservice;





























import static android.view.inputmethod.TextBoundsInfoResult.CODE_CANCELLED;





























import android.annotation.AnyThread;















import android.annotation.CallbackExecutor;















import android.annotation.NonNull;















import android.annotation.Nullable;













import android.graphics.RectF;















import android.os.Bundle;















import android.os.RemoteException;















import android.os.ResultReceiver;









@@ -40,6 +43,8 @@ import android.view.inputmethod.SelectGesture;













import android.view.inputmethod.SelectRangeGesture;















import android.view.inputmethod.SurroundingText;















import android.view.inputmethod.TextAttribute;













import android.view.inputmethod.TextBoundsInfo;













import android.view.inputmethod.TextBoundsInfoResult;































import com.android.internal.infra.AndroidFuture;















import com.android.internal.inputmethod.IRemoteInputConnection;









@@ -47,6 +52,7 @@ import com.android.internal.inputmethod.InputConnectionCommandHeader;





























import java.util.Objects;















import java.util.concurrent.Executor;













import java.util.function.Consumer;















import java.util.function.IntConsumer;































/**










@@ -97,6 +103,44 @@ final class IRemoteInputConnectionInvoker {













        }















    };





























    /**













     * Subclass of {@link ResultReceiver} used by













     * {@link #requestTextBoundsInfo(RectF, Executor, Consumer)} for providing













     * callback.













     */













    private static final class TextBoundsInfoResultReceiver extends ResultReceiver {













        @Nullable













        private Consumer<TextBoundsInfoResult> mConsumer;













        @Nullable













        private Executor mExecutor;



























        TextBoundsInfoResultReceiver(@NonNull Executor executor,













                @NonNull Consumer<TextBoundsInfoResult> consumer) {













            super(null);













            mExecutor = executor;













            mConsumer = consumer;













        }



























        @Override













        protected void onReceiveResult(@TextBoundsInfoResult.ResultCode int resultCode,













                @Nullable Bundle resultData) {













            synchronized (this) {













                if (mExecutor != null && mConsumer != null) {













                    final TextBoundsInfoResult textBoundsInfoResult = new TextBoundsInfoResult(













                            resultCode, TextBoundsInfo.createFromBundle(resultData));













                    mExecutor.execute(() -> mConsumer.accept(textBoundsInfoResult));













                    // provide callback only once.













                    clear();













                }













            }













        }



























        private void clear() {













            mExecutor = null;













            mConsumer = null;













        }













    }





























    /**















     * Creates a new instance of {@link IRemoteInputConnectionInvoker} for the given















     * {@link IRemoteInputConnection}.










@@ -735,6 +779,28 @@ final class IRemoteInputConnectionInvoker {













        return future;















    }





























    /**













     * Invokes {@link IRemoteInputConnection#requestTextBoundsInfo(InputConnectionCommandHeader,













     * RectF, ResultReceiver)}













     * @param rectF {@code rectF} parameter to be passed.













     * @param executor {@code Executor} parameter to be passed.













     * @param consumer {@code Consumer} parameter to be passed.













     */













    @AnyThread













    public void requestTextBoundsInfo(













            @NonNull RectF rectF, @NonNull @CallbackExecutor Executor executor,













            @NonNull Consumer<TextBoundsInfoResult> consumer) {













        Objects.requireNonNull(executor);













        Objects.requireNonNull(consumer);



























        final ResultReceiver resultReceiver = new TextBoundsInfoResultReceiver(executor, consumer);













        try {













            mConnection.requestTextBoundsInfo(createHeader(), rectF, resultReceiver);













        } catch (RemoteException e) {













            executor.execute(() -> consumer.accept(new TextBoundsInfoResult(CODE_CANCELLED)));













        }













    }





























    /**















     * Invokes {@link IRemoteInputConnection#commitContent(InputConnectionCommandHeader,















     * InputContentInfo, int, Bundle, AndroidFuture)}.

































core/java/android/inputmethodservice/RemoteInputConnection.java



+10
−0




























Original line number
Diff line number
Diff line








@@ -21,6 +21,7 @@ import android.annotation.CallbackExecutor;













import android.annotation.IntRange;















import android.annotation.NonNull;















import android.annotation.Nullable;













import android.graphics.RectF;















import android.os.Bundle;















import android.os.Handler;















import android.util.Log;









@@ -34,6 +35,7 @@ import android.view.inputmethod.InputConnection;













import android.view.inputmethod.InputContentInfo;















import android.view.inputmethod.SurroundingText;















import android.view.inputmethod.TextAttribute;













import android.view.inputmethod.TextBoundsInfoResult;































import com.android.internal.inputmethod.CancellationGroup;















import com.android.internal.inputmethod.CompletableFutureUtil;









@@ -44,6 +46,7 @@ import com.android.internal.inputmethod.InputConnectionProtoDumper;













import java.lang.ref.WeakReference;















import java.util.concurrent.CompletableFuture;















import java.util.concurrent.Executor;













import java.util.function.Consumer;















import java.util.function.IntConsumer;































/**










@@ -459,6 +462,13 @@ final class RemoteInputConnection implements InputConnection {













                mCancellationGroup, MAX_WAIT_TIME_MILLIS);















    }





























    @AnyThread













    public void requestTextBoundsInfo(













            @NonNull RectF rectF, @NonNull @CallbackExecutor Executor executor,













            @NonNull Consumer<TextBoundsInfoResult> consumer) {













        mInvoker.requestTextBoundsInfo(rectF, executor, consumer);













    }





























    @AnyThread















    public Handler getHandler() {















        // Nothing should happen when called from input method.

































core/java/android/text/SegmentFinder.java



+147
−0




























Original line number
Diff line number
Diff line








@@ -19,6 +19,13 @@ package android.text;













import android.annotation.IntRange;















import android.graphics.RectF;





























import androidx.annotation.NonNull;



























import com.android.internal.util.Preconditions;



























import java.util.Arrays;













import java.util.Objects;





























/**















 * Finds text segment boundaries within text. Subclasses can implement different types of text















 * segments. Grapheme clusters and words are examples of possible text segments. These are










@@ -63,4 +70,144 @@ public abstract class SegmentFinder {













     * character offset, or {@code DONE} if there are none.















     */















    public abstract int nextEndBoundary(@IntRange(from = 0) int offset);



























    /**













     * The default {@link SegmentFinder} implementation based on given segment ranges.













     */













    public static class DefaultSegmentFinder extends SegmentFinder {













        private final int[] mSegments;



























        /**













         * Create a SegmentFinder with segments stored in an array, where i-th segment's start is













         * stored at segments[2 * i] and end is stored at segments[2 * i + 1] respectively.













         *













         * <p> It is required that segments do not overlap, and are already sorted by their start













         * indices. </p>













         * @param segments the array that stores the segment ranges.













         * @throws IllegalArgumentException if the given segments array's length is not even; the













         * given segments are not sorted or there are segments overlap with others.













         */













        public DefaultSegmentFinder(@NonNull int[] segments) {













            checkSegmentsValid(segments);













            mSegments = segments;













        }



























        /** {@inheritDoc} */













        @Override













        public int previousStartBoundary(@IntRange(from = 0) int offset) {













            return findPrevious(offset, /* isStart = */ true);













        }



























        /** {@inheritDoc} */













        @Override













        public int previousEndBoundary(@IntRange(from = 0) int offset) {













            return findPrevious(offset, /* isStart = */ false);













        }



























        /** {@inheritDoc} */













        @Override













        public int nextStartBoundary(@IntRange(from = 0) int offset) {













            return findNext(offset, /* isStart = */ true);













        }



























        /** {@inheritDoc} */













        @Override













        public int nextEndBoundary(@IntRange(from = 0) int offset) {













            return findNext(offset, /* isStart = */ false);













        }



























        private int findNext(int offset, boolean isStart) {













            if (offset < 0) return DONE;













            if (mSegments.length < 1 || offset > mSegments[mSegments.length - 1]) return DONE;



























            if (offset < mSegments[0]) {













                return isStart ? mSegments[0] : mSegments[1];













            }



























            int index = Arrays.binarySearch(mSegments, offset);













            if (index >= 0) {













                // mSegments may have duplicate elements (The previous segments end equals













                // to the following segments start.) Move the index forwards since we are searching













                // for the next segment.













                if (index + 1 < mSegments.length && mSegments[index + 1] == offset) {













                    index = index + 1;













                }













                // Point the index to the first segment boundary larger than the given offset.













                index += 1;













            } else {













                // binarySearch returns the insertion point, it's the first segment boundary larger













                // than the given offset.













                index = -(index + 1);













            }













            if (index >= mSegments.length) return DONE;



























            //  +---------------------------------------+













            //  |               | isStart   | isEnd     |













            //  |---------------+-----------+-----------|













            //  | indexIsStart  | index     | index + 1 |













            //  |---------------+-----------+-----------|













            //  | indexIsEnd    | index + 1 | index     |













            //  +---------------------------------------+













            boolean indexIsStart = index % 2 == 0;













            if (isStart != indexIsStart) {













                return (index + 1 < mSegments.length) ? mSegments[index + 1] : DONE;













            }













            return mSegments[index];













        }



























        private int findPrevious(int offset, boolean isStart) {













            if (mSegments.length < 1 || offset < mSegments[0]) return DONE;



























            if (offset > mSegments[mSegments.length - 1]) {













                return isStart ? mSegments[mSegments.length - 2] : mSegments[mSegments.length - 1];













            }



























            int index = Arrays.binarySearch(mSegments, offset);













            if (index >= 0) {













                // mSegments may have duplicate elements (when the previous segments end equal













                // to the following segments start). Move the index backwards since we are searching













                // for the previous segment.













                if (index > 0 && mSegments[index - 1] == offset) {













                    index = index - 1;













                }













                // Point the index to the first segment boundary smaller than the given offset.













                index -= 1;













            } else {













                // binarySearch returns the insertion point, insertionPoint - 1 is the first













                // segment boundary smaller than the given offset.













                index = -(index + 1) - 1;













            }













            if (index < 0) return DONE;



























            //  +---------------------------------------+













            //  |               | isStart   | isEnd     |













            //  |---------------+-----------+-----------|













            //  | indexIsStart  | index     | index - 1 |













            //  |---------------+-----------+-----------|













            //  | indexIsEnd    | index - 1 | index     |













            //  +---------------------------------------+













            boolean indexIsStart = index % 2 == 0;













            if (isStart != indexIsStart) {













                return (index > 0) ? mSegments[index - 1] : DONE;













            }













            return mSegments[index];













        }



























        private static void checkSegmentsValid(int[] segments) {













            Objects.requireNonNull(segments);













            Preconditions.checkArgument(segments.length % 2 == 0,













                    "the length of segments must be even");













            if (segments.length == 0) return;













            int lastSegmentEnd = Integer.MIN_VALUE;













            for (int index = 0; index < segments.length; index += 2) {













                if (segments[index] < lastSegmentEnd) {













                    throw new IllegalArgumentException("segments can't overlap");













                }













                if (segments[index] >= segments[index + 1]) {













                    throw new IllegalArgumentException("the segment range can't be empty");













                }













                lastSegmentEnd = segments[index + 1];













            }













        }













    }















}
























core/java/android/view/inputmethod/InputConnection.java



+39
−0




























Original line number
Diff line number
Diff line








@@ -16,11 +16,14 @@





























package android.view.inputmethod;





























import static android.view.inputmethod.TextBoundsInfoResult.CODE_UNSUPPORTED;





























import android.annotation.CallbackExecutor;















import android.annotation.IntDef;















import android.annotation.IntRange;















import android.annotation.NonNull;















import android.annotation.Nullable;













import android.graphics.RectF;















import android.inputmethodservice.InputMethodService;















import android.os.Bundle;















import android.os.Handler;









@@ -32,7 +35,9 @@ import com.android.internal.util.Preconditions;





























import java.lang.annotation.Retention;















import java.lang.annotation.RetentionPolicy;













import java.util.Objects;















import java.util.concurrent.Executor;













import java.util.function.Consumer;















import java.util.function.IntConsumer;































/**










@@ -1205,6 +1210,40 @@ public interface InputConnection {













        return false;















    }











































    /**













     * Called by input method to request the {@link TextBoundsInfo} for a range of text which is













     * covered by or in vicinity of the given {@code RectF}. It can be used as a supplementary













     * method to implement the handwriting gesture API -













     * {@link #performHandwritingGesture(HandwritingGesture, Executor, IntConsumer)}.













     *













     * <p><strong>Editor authors</strong>: It's preferred that the editor returns a













     * {@link TextBoundsInfo} of all the text lines whose bounds intersect with the given













     * {@code rectF}.













     * </p>













     *













     * <p><strong>IME authors</strong>: This method is expensive when the text is long. Please













     * consider that both the text bounds computation and IPC round-trip to send the data are time













     * consuming. It's preferable to only request text bounds in smaller areas.













     * </p>













     *













     * @param rectF the interested area where the text bounds are requested, in the screen













     *              coordinates.













     * @param executor the executor to run the callback.













     * @param consumer the callback invoked by editor to return the result. It must return a













     *                 non-null object.













     *













     * @see TextBoundsInfo













     * @see android.view.inputmethod.TextBoundsInfoResult













     */













    default void requestTextBoundsInfo(













            @NonNull RectF rectF, @NonNull @CallbackExecutor Executor executor,













            @NonNull Consumer<TextBoundsInfoResult> consumer) {













        Objects.requireNonNull(executor);













        Objects.requireNonNull(consumer);













        executor.execute(() -> consumer.accept(new TextBoundsInfoResult(CODE_UNSUPPORTED)));













    }





























    /**















     * Called by the system to enable application developers to specify a dedicated thread on which















     * {@link InputConnection} methods are called back.