Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit ae6688b0 authored by Dianne Hackborn's avatar Dianne Hackborn
Browse files

Update voice interaction layer for new UI design. 

Can switch from a pure overlay at the top of the screen,
to interactive mode with the voice UI drawing at the bottom
and pushing its target activity up like an IME.

Add mechanism to get assist data to the voice interaction UI.

Add some basic visualization of the assist data, outlining
where on the screen we have text.

Add a test ACTION_ASSIST handler, which can propagate the
assist data it gets to the voice interaction session so
you can see what kind of data we are getting from different
apps.

Change-Id: I18312fe1601d7926d1fb96a817638d60f6263771
parent bc26d2ea

api/current.txt

+5 −2
Original line number Diff line number Diff line
@@ -3937,6 +3937,7 @@ package android.app { 














  public final class AssistData implements android.os.Parcelable {















    method public int describeContents();













    method public android.content.ComponentName getActivityComponent();















    method public static android.app.AssistData getAssistData(android.os.Bundle);















    method public void getWindowAt(int, android.app.AssistData.ViewNode);















    method public int getWindowCount();










@@ -27644,9 +27645,10 @@ package android.service.voice {













    method public android.os.IBinder onBind(android.content.Intent);















    method public void onReady();















    method public void onShutdown();













    method public void startSession(android.os.Bundle);













    method public void startSession(android.os.Bundle, int);















    field public static final java.lang.String SERVICE_INTERFACE = "android.service.voice.VoiceInteractionService";















    field public static final java.lang.String SERVICE_META_DATA = "android.voice_interaction";













    field public static final int START_WITH_ASSIST = 1; // 0x1















  }





























  public abstract class VoiceInteractionSession implements android.view.KeyEvent.Callback {









@@ -27664,10 +27666,11 @@ package android.service.voice {













    method public void onCompleteVoice(android.service.voice.VoiceInteractionSession.Caller, android.service.voice.VoiceInteractionSession.Request, java.lang.CharSequence, android.os.Bundle);















    method public void onComputeInsets(android.service.voice.VoiceInteractionSession.Insets);















    method public abstract void onConfirm(android.service.voice.VoiceInteractionSession.Caller, android.service.voice.VoiceInteractionSession.Request, java.lang.CharSequence, android.os.Bundle);













    method public void onCreate(android.os.Bundle);













    method public void onCreate(android.os.Bundle, int);















    method public android.view.View onCreateContentView();















    method public void onDestroy();















    method public boolean[] onGetSupportedCommands(android.service.voice.VoiceInteractionSession.Caller, java.lang.String[]);













    method public void onHandleAssist(android.os.Bundle);















    method public boolean onKeyDown(int, android.view.KeyEvent);















    method public boolean onKeyLongPress(int, android.view.KeyEvent);















    method public boolean onKeyMultiple(int, int, android.view.KeyEvent);

























api/system-current.txt



+5
−2






























Original line number
Diff line number
Diff line








@@ -4027,6 +4027,7 @@ package android.app {



























  public final class AssistData implements android.os.Parcelable {















    method public int describeContents();













    method public android.content.ComponentName getActivityComponent();















    method public static android.app.AssistData getAssistData(android.os.Bundle);















    method public void getWindowAt(int, android.app.AssistData.ViewNode);















    method public int getWindowCount();










@@ -29331,9 +29332,10 @@ package android.service.voice {













    method public android.os.IBinder onBind(android.content.Intent);















    method public void onReady();















    method public void onShutdown();













    method public void startSession(android.os.Bundle);













    method public void startSession(android.os.Bundle, int);















    field public static final java.lang.String SERVICE_INTERFACE = "android.service.voice.VoiceInteractionService";















    field public static final java.lang.String SERVICE_META_DATA = "android.voice_interaction";













    field public static final int START_WITH_ASSIST = 1; // 0x1















  }





























  public abstract class VoiceInteractionSession implements android.view.KeyEvent.Callback {









@@ -29351,10 +29353,11 @@ package android.service.voice {













    method public void onCompleteVoice(android.service.voice.VoiceInteractionSession.Caller, android.service.voice.VoiceInteractionSession.Request, java.lang.CharSequence, android.os.Bundle);















    method public void onComputeInsets(android.service.voice.VoiceInteractionSession.Insets);















    method public abstract void onConfirm(android.service.voice.VoiceInteractionSession.Caller, android.service.voice.VoiceInteractionSession.Request, java.lang.CharSequence, android.os.Bundle);













    method public void onCreate(android.os.Bundle);













    method public void onCreate(android.os.Bundle, int);















    method public android.view.View onCreateContentView();















    method public void onDestroy();















    method public boolean[] onGetSupportedCommands(android.service.voice.VoiceInteractionSession.Caller, java.lang.String[]);













    method public void onHandleAssist(android.os.Bundle);















    method public boolean onKeyDown(int, android.view.KeyEvent);















    method public boolean onKeyLongPress(int, android.view.KeyEvent);















    method public boolean onKeyMultiple(int, int, android.view.KeyEvent);

























core/java/android/app/ActivityManagerNative.java



+23
−0






























Original line number
Diff line number
Diff line








@@ -51,6 +51,7 @@ import android.text.TextUtils;













import android.util.Log;















import android.util.Singleton;















import com.android.internal.app.IVoiceInteractor;













import com.android.internal.os.IResultReceiver;































import java.util.ArrayList;















import java.util.List;










@@ -2114,6 +2115,15 @@ public abstract class ActivityManagerNative extends Binder implements IActivityM













            return true;















        }





























        case REQUEST_ASSIST_CONTEXT_EXTRAS_TRANSACTION: {













            data.enforceInterface(IActivityManager.descriptor);













            int requestType = data.readInt();













            IResultReceiver receiver = IResultReceiver.Stub.asInterface(data.readStrongBinder());













            requestAssistContextExtras(requestType, receiver);













            reply.writeNoException();













            return true;













        }





























        case REPORT_ASSIST_CONTEXT_EXTRAS_TRANSACTION: {















            data.enforceInterface(IActivityManager.descriptor);















            IBinder token = data.readStrongBinder();










@@ -5146,6 +5156,19 @@ class ActivityManagerProxy implements IActivityManager













        return res;















    }





























    public void requestAssistContextExtras(int requestType, IResultReceiver receiver)













            throws RemoteException {













        Parcel data = Parcel.obtain();













        Parcel reply = Parcel.obtain();













        data.writeInterfaceToken(IActivityManager.descriptor);













        data.writeInt(requestType);













        data.writeStrongBinder(receiver.asBinder());













        mRemote.transact(REQUEST_ASSIST_CONTEXT_EXTRAS_TRANSACTION, data, reply, 0);













        reply.readException();













        data.recycle();













        reply.recycle();













    }





























    public void reportAssistContextExtras(IBinder token, Bundle extras)















            throws RemoteException {















        Parcel data = Parcel.obtain();

































core/java/android/app/AssistData.java



+14
−2






























Original line number
Diff line number
Diff line








@@ -16,6 +16,7 @@





























package android.app;





























import android.content.ComponentName;















import android.graphics.Rect;















import android.os.Bundle;















import android.os.Parcel;










@@ -44,6 +45,8 @@ final public class AssistData implements Parcelable {













     */















    public static final String ASSIST_KEY = "android:assist";





























    final ComponentName mActivityComponent;





























    final ArrayList<ViewNodeImpl> mRootViews = new ArrayList<>();































    ViewAssistDataImpl mTmpViewAssistDataImpl = new ViewAssistDataImpl();










@@ -400,6 +403,7 @@ final public class AssistData implements Parcelable {













    }































    AssistData(Activity activity) {













        mActivityComponent = activity.getComponentName();















        ArrayList<ViewRootImpl> views = WindowManagerGlobal.getInstance().getRootViews(















                activity.getActivityToken());















        for (int i=0; i<views.size(); i++) {









@@ -414,6 +418,7 @@ final public class AssistData implements Parcelable {













    }































    AssistData(Parcel in) {













        mActivityComponent = ComponentName.readFromParcel(in);















        final int N = in.readInt();















        for (int i=0; i<N; i++) {















            mRootViews.add(new ViewNodeImpl(in));









@@ -421,7 +426,9 @@ final public class AssistData implements Parcelable {













        //dump();















    }





























    void dump() {













    /** @hide */













    public void dump() {













        Log.i(TAG, "Activity: " + mActivityComponent.flattenToShortString());















        ViewNode node = new ViewNode();















        final int N = getWindowCount();















        for (int i=0; i<N; i++) {










@@ -476,6 +483,10 @@ final public class AssistData implements Parcelable {













        return assistBundle.getParcelable(ASSIST_KEY);















    }





























    public ComponentName getActivityComponent() {













        return mActivityComponent;













    }





























    /**















     * Return the number of window contents that have been collected in this assist data.















     */









@@ -498,6 +509,7 @@ final public class AssistData implements Parcelable {





























    public void writeToParcel(Parcel out, int flags) {















        int start = out.dataPosition();













        ComponentName.writeToParcel(mActivityComponent, out);















        final int N = mRootViews.size();















        out.writeInt(N);















        for (int i=0; i<N; i++) {

































core/java/android/app/IActivityManager.java



+5
−0






























Original line number
Diff line number
Diff line








@@ -51,6 +51,7 @@ import android.os.RemoteException;













import android.os.StrictMode;















import android.service.voice.IVoiceInteractionSession;















import com.android.internal.app.IVoiceInteractor;













import com.android.internal.os.IResultReceiver;































import java.util.List;


























@@ -419,6 +420,9 @@ public interface IActivityManager extends IInterface {





























    public Bundle getAssistContextExtras(int requestType) throws RemoteException;





























    public void requestAssistContextExtras(int requestType, IResultReceiver receiver)













            throws RemoteException;





























    public void reportAssistContextExtras(IBinder token, Bundle extras) throws RemoteException;































    public boolean launchAssistIntent(Intent intent, int requestType, String hint, int userHandle)










@@ -804,4 +808,5 @@ public interface IActivityManager extends IInterface {













    int CREATE_STACK_ON_DISPLAY = IBinder.FIRST_CALL_TRANSACTION+281;















    int GET_FOCUSED_STACK_ID_TRANSACTION = IBinder.FIRST_CALL_TRANSACTION+282;















    int SET_TASK_RESIZEABLE_TRANSACTION = IBinder.FIRST_CALL_TRANSACTION+283;













    int REQUEST_ASSIST_CONTEXT_EXTRAS_TRANSACTION = IBinder.FIRST_CALL_TRANSACTION+284;















}