Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 83ed20d2 authored by Charles Chen's avatar Charles Chen
Browse files

Add system change to enable stop/start recognition 

Add necessary function to enable system server to start VQDS and change
egression states with the callback toggleing.

Bug: 261783496
Test: atest CtsVoiceInteractionTestCases
Change-Id: Id5565eb8138ad48c5141013e725863041b431410
parent c9e0fd55

core/java/android/service/voice/IVisualQueryDetectionVoiceInteractionCallback.aidl

0 → 100644
+48 −0
Original line number Original line Diff line number Diff line 
/*

 * Copyright (C) 2023 The Android Open Source Project

 *

 * Licensed under the Apache License, Version 2.0 (the "License");

 * you may not use this file except in compliance with the License.

 * You may obtain a copy of the License at

 *

 *      http://www.apache.org/licenses/LICENSE-2.0

 *

 * Unless required by applicable law or agreed to in writing, software

 * distributed under the License is distributed on an "AS IS" BASIS,

 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

 * See the License for the specific language governing permissions and

 * limitations under the License.

 */



package android.service.voice;



import android.media.AudioFormat;



/**

 * Callback for returning the detected result from the VisualQueryDetectionService.

 *

 * @hide

 */

oneway interface IVisualQueryDetectionVoiceInteractionCallback {



    /**

     * Called when the detected query is streamed

     */

    void onQueryDetected(in String partialQuery);



    /**

     * Called when the detected result is valid.

     */

    void onQueryFinished();



    /**

     * Called when the detected result is invalid.

     */

    void onQueryRejected();



    /**

     * Called when the detection fails due to an error.

     */

    void onError();



}

core/java/com/android/internal/app/IVoiceInteractionManagerService.aidl

+5 −0
Original line number Original line Diff line number Diff line
@@ -27,6 +27,7 @@ import android.os.ParcelFileDescriptor; 
import android.os.PersistableBundle;

import android.os.PersistableBundle;

import android.os.RemoteCallback;

import android.os.RemoteCallback;

import android.os.SharedMemory;

import android.os.SharedMemory;

import android.service.voice.IVisualQueryDetectionVoiceInteractionCallback;

import android.service.voice.IMicrophoneHotwordDetectionVoiceInteractionCallback;

import android.service.voice.IMicrophoneHotwordDetectionVoiceInteractionCallback;

import android.service.voice.IVoiceInteractionService;

import android.service.voice.IVoiceInteractionService;

import android.service.voice.IVoiceInteractionSession;

import android.service.voice.IVoiceInteractionSession;
@@ -299,6 +300,10 @@ interface IVoiceInteractionManagerService { 
     */

     */

    void shutdownHotwordDetectionService();

    void shutdownHotwordDetectionService();





    void startPerceiving(in IVisualQueryDetectionVoiceInteractionCallback callback);



    void stopPerceiving();



    void startListeningFromMic(

    void startListeningFromMic(

        in AudioFormat audioFormat,

        in AudioFormat audioFormat,

        in IMicrophoneHotwordDetectionVoiceInteractionCallback callback);

        in IMicrophoneHotwordDetectionVoiceInteractionCallback callback);

services/voiceinteraction/java/com/android/server/voiceinteraction/HotwordDetectionConnection.java

+29 −0
Original line number Original line Diff line number Diff line
@@ -52,6 +52,7 @@ import android.service.voice.HotwordDetectionService; 
import android.service.voice.HotwordDetector;

import android.service.voice.HotwordDetector;

import android.service.voice.IMicrophoneHotwordDetectionVoiceInteractionCallback;

import android.service.voice.IMicrophoneHotwordDetectionVoiceInteractionCallback;

import android.service.voice.ISandboxedDetectionService;

import android.service.voice.ISandboxedDetectionService;

import android.service.voice.IVisualQueryDetectionVoiceInteractionCallback;

import android.service.voice.VisualQueryDetectionService;

import android.service.voice.VisualQueryDetectionService;

import android.service.voice.VoiceInteractionManagerInternal.HotwordDetectionServiceIdentity;

import android.service.voice.VoiceInteractionManagerInternal.HotwordDetectionServiceIdentity;

import android.speech.IRecognitionServiceManager;

import android.speech.IRecognitionServiceManager;
@@ -290,6 +291,34 @@ final class HotwordDetectionConnection { 
        session.startListeningFromMicLocked(audioFormat, callback);

        session.startListeningFromMicLocked(audioFormat, callback);

    }

    }





    /**

     * This method is only used by VisualQueryDetector.

     */

    void startPerceivingLocked(IVisualQueryDetectionVoiceInteractionCallback callback) {

        if (DEBUG) {

            Slog.d(TAG, "startPerceivingLocked");

        }

        final VisualQueryDetectorSession session = getVisualQueryDetectorSessionLocked();

        if (session == null) {

            return;

        }

        session.startPerceivingLocked(callback);

    }



    /**

     * This method is only used by VisaulQueryDetector.

     */

    void stopPerceivingLocked() {

        if (DEBUG) {

            Slog.d(TAG, "stopPerceivingLocked");

        }

        final VisualQueryDetectorSession session = getVisualQueryDetectorSessionLocked();

        if (session == null) {

            return;

        }

        session.stopPerceivingLocked();

    }



    public void startListeningFromExternalSourceLocked(

    public void startListeningFromExternalSourceLocked(

            ParcelFileDescriptor audioStream,

            ParcelFileDescriptor audioStream,

            AudioFormat audioFormat,

            AudioFormat audioFormat,

services/voiceinteraction/java/com/android/server/voiceinteraction/VisualQueryDetectorSession.java

+80 −0
Original line number Original line Diff line number Diff line
@@ -24,13 +24,18 @@ import android.media.permission.Identity; 
import android.os.IBinder;

import android.os.IBinder;

import android.os.ParcelFileDescriptor;

import android.os.ParcelFileDescriptor;

import android.os.PersistableBundle;

import android.os.PersistableBundle;

import android.os.RemoteException;

import android.os.SharedMemory;

import android.os.SharedMemory;

import android.service.voice.IDetectorSessionVisualQueryDetectionCallback;

import android.service.voice.IMicrophoneHotwordDetectionVoiceInteractionCallback;

import android.service.voice.IMicrophoneHotwordDetectionVoiceInteractionCallback;

import android.service.voice.ISandboxedDetectionService;

import android.service.voice.IVisualQueryDetectionVoiceInteractionCallback;

import android.util.Slog;

import android.util.Slog;





import com.android.internal.app.IHotwordRecognitionStatusCallback;

import com.android.internal.app.IHotwordRecognitionStatusCallback;





import java.io.PrintWriter;

import java.io.PrintWriter;

import java.util.Objects;

import java.util.concurrent.ScheduledExecutorService;

import java.util.concurrent.ScheduledExecutorService;





/**

/**
@@ -44,6 +49,8 @@ import java.util.concurrent.ScheduledExecutorService; 
final class VisualQueryDetectorSession extends DetectorSession {

final class VisualQueryDetectorSession extends DetectorSession {





    private static final String TAG = "VisualQueryDetectorSession";

    private static final String TAG = "VisualQueryDetectorSession";

    private boolean mEgressingData;

    private boolean mQueryStreaming;





    //TODO(b/261783819): Determines actual functionalities, e.g., startRecognition etc.

    //TODO(b/261783819): Determines actual functionalities, e.g., startRecognition etc.

    VisualQueryDetectorSession(

    VisualQueryDetectorSession(
@@ -55,6 +62,8 @@ final class VisualQueryDetectorSession extends DetectorSession { 
        super(remoteService, lock, context, token, callback,

        super(remoteService, lock, context, token, callback,

                voiceInteractionServiceUid, voiceInteractorIdentity, scheduledExecutorService,

                voiceInteractionServiceUid, voiceInteractorIdentity, scheduledExecutorService,

                logging);

                logging);

        mEgressingData = false;

        mQueryStreaming = false;

    }

    }





    @Override

    @Override
@@ -65,6 +74,77 @@ final class VisualQueryDetectorSession extends DetectorSession { 
        //TODO(b/261783819): Starts detection in VisualQueryDetectionService.

        //TODO(b/261783819): Starts detection in VisualQueryDetectionService.

    }

    }





    @SuppressWarnings("GuardedBy")

    void startPerceivingLocked(IVisualQueryDetectionVoiceInteractionCallback callback) {

        if (DEBUG) {

            Slog.d(TAG, "startPerceivingLocked");

        }



        IDetectorSessionVisualQueryDetectionCallback internalCallback =

                new IDetectorSessionVisualQueryDetectionCallback.Stub(){



            @Override

            public void onAttentionGained() {

                Slog.v(TAG, "BinderCallback#onAttentionGained");

                //TODO check to see if there is an active SysUI listener registered

                mEgressingData = true;

            }



            @Override

            public void onAttentionLost() {

                Slog.v(TAG, "BinderCallback#onAttentionLost");

                //TODO check to see if there is an active SysUI listener registered

                mEgressingData = false;

            }



            @Override

            public void onQueryDetected(@NonNull String partialQuery) throws RemoteException {

                Objects.requireNonNull(partialQuery);

                Slog.v(TAG, "BinderCallback#onQueryDetected");

                if (!mEgressingData) {

                    Slog.v(TAG, "Query should not be egressed within the unattention state.");

                    return;

                }

                mQueryStreaming = true;

                callback.onQueryDetected(partialQuery);

                Slog.i(TAG, "Egressed from visual query detection process.");

            }



            @Override

            public void onQueryFinished() throws RemoteException {

                Slog.v(TAG, "BinderCallback#onQueryFinished");

                if (!mQueryStreaming) {

                    Slog.v(TAG, "Query streaming state signal FINISHED is block since there is"

                            + " no active query being streamed.");

                    return;

                }

                callback.onQueryFinished();

                mQueryStreaming = false;

            }



            @Override

            public void onQueryRejected() throws RemoteException {

                Slog.v(TAG, "BinderCallback#onQueryRejected");

                if (!mQueryStreaming) {

                    Slog.v(TAG, "Query streaming state signal REJECTED is block since there is"

                            + " no active query being streamed.");

                    return;

                }

                callback.onQueryRejected();

                mQueryStreaming = false;

            }

        };

        mRemoteDetectionService.run(service -> service.detectWithVisualSignals(internalCallback));

    }



    @SuppressWarnings("GuardedBy")

    void stopPerceivingLocked() {

        if (DEBUG) {

            Slog.d(TAG, "stopPerceivingLocked");

        }

        mRemoteDetectionService.run(ISandboxedDetectionService::stopDetection);

    }



    @Override

    @Override

     void startListeningFromExternalSourceLocked(

     void startListeningFromExternalSourceLocked(

            ParcelFileDescriptor audioStream,

            ParcelFileDescriptor audioStream,

services/voiceinteraction/java/com/android/server/voiceinteraction/VoiceInteractionManagerService.java

+41 −0
Original line number Original line Diff line number Diff line
@@ -72,6 +72,7 @@ import android.os.Trace; 
import android.os.UserHandle;

import android.os.UserHandle;

import android.provider.Settings;

import android.provider.Settings;

import android.service.voice.IMicrophoneHotwordDetectionVoiceInteractionCallback;

import android.service.voice.IMicrophoneHotwordDetectionVoiceInteractionCallback;

import android.service.voice.IVisualQueryDetectionVoiceInteractionCallback;

import android.service.voice.IVoiceInteractionSession;

import android.service.voice.IVoiceInteractionSession;

import android.service.voice.VoiceInteractionManagerInternal;

import android.service.voice.VoiceInteractionManagerInternal;

import android.service.voice.VoiceInteractionService;

import android.service.voice.VoiceInteractionService;
@@ -1313,6 +1314,46 @@ public class VoiceInteractionManagerService extends SystemService { 
            }

            }

        }

        }





        @Override

        public void startPerceiving(

                IVisualQueryDetectionVoiceInteractionCallback callback)

                throws RemoteException {

            enforceCallingPermission(Manifest.permission.RECORD_AUDIO);

            enforceCallingPermission(Manifest.permission.CAMERA);

            synchronized (this) {

                enforceIsCurrentVoiceInteractionService();



                if (mImpl == null) {

                    Slog.w(TAG, "startPerceiving without running voice interaction service");

                    return;

                }

                final long caller = Binder.clearCallingIdentity();

                try {

                    mImpl.startPerceivingLocked(callback);

                } finally {

                    Binder.restoreCallingIdentity(caller);

                }

            }

        }



        @Override

        public void stopPerceiving() throws RemoteException {

            synchronized (this) {

                enforceIsCurrentVoiceInteractionService();



                if (mImpl == null) {

                    Slog.w(TAG, "stopPerceiving without running voice interaction service");

                    return;

                }

                final long caller = Binder.clearCallingIdentity();

                try {

                    mImpl.stopPerceivingLocked();

                } finally {

                    Binder.restoreCallingIdentity(caller);

                }

            }

        }



        @Override

        @Override

        public void startListeningFromMic(

        public void startListeningFromMic(

                AudioFormat audioFormat,

                AudioFormat audioFormat,