Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 25272aab authored by Nicholas Ambur's avatar Nicholas Ambur
Browse files

Add capture timestamp field to AIDL recognition event 

- Add RecognitionEventSys to encapsulate HAL AIDL type plus framework extras

Bug: 265852186
Test: atest CtsVoiceInteractionHostTestCases
Test: atest FrameworksVoiceInteractionTests
Change-Id: Idd3dc44b89ea7cc8692ed5d9e6b7f4bb753a9eb6
parent 3ad0e9fc

core/java/android/hardware/soundtrigger/ConversionUtil.java

+26 −20
Original line number Diff line number Diff line
@@ -32,10 +32,12 @@ import android.media.soundtrigger.RecognitionConfig; 
import android.media.soundtrigger.RecognitionEvent;

import android.media.soundtrigger.RecognitionMode;

import android.media.soundtrigger.SoundModel;

import android.media.soundtrigger_middleware.PhraseRecognitionEventSys;

import android.media.soundtrigger_middleware.RecognitionEventSys;

import android.media.soundtrigger_middleware.SoundTriggerModuleDescriptor;

import android.os.ParcelFileDescriptor;

import android.system.ErrnoException;

import android.os.SharedMemory;

import android.system.ErrnoException;



import java.nio.ByteBuffer;

import java.util.Arrays;
@@ -219,36 +221,40 @@ public class ConversionUtil { 
        return new SoundTrigger.ConfidenceLevel(apiLevel.userId, apiLevel.levelPercent);

    }



    public static SoundTrigger.RecognitionEvent aidl2apiRecognitionEvent(

            int modelHandle, int captureSession, RecognitionEvent aidlEvent) {

    public static SoundTrigger.RecognitionEvent aidl2apiRecognitionEvent(int modelHandle,

            int captureSession, RecognitionEventSys aidlEvent) {

        RecognitionEvent recognitionEvent = aidlEvent.recognitionEvent;

        // The API recognition event doesn't allow for a null audio format, even though it doesn't

        // always make sense. We thus replace it with a default.

        AudioFormat audioFormat = aidl2apiAudioFormatWithDefault(aidlEvent.audioConfig,

        AudioFormat audioFormat = aidl2apiAudioFormatWithDefault(recognitionEvent.audioConfig,

                true /*isInput*/);

        // TODO(b/265852186) propagate a timestamp from aidl interfaces

        return new SoundTrigger.GenericRecognitionEvent(aidlEvent.status, modelHandle,

                aidlEvent.captureAvailable, captureSession, aidlEvent.captureDelayMs,

                aidlEvent.capturePreambleMs, aidlEvent.triggerInData, audioFormat, aidlEvent.data,

                aidlEvent.recognitionStillActive, -1 /* halEventReceivedMillis */);

        return new SoundTrigger.GenericRecognitionEvent(recognitionEvent.status, modelHandle,

                recognitionEvent.captureAvailable, captureSession, recognitionEvent.captureDelayMs,

                recognitionEvent.capturePreambleMs, recognitionEvent.triggerInData, audioFormat,

                recognitionEvent.data,

                recognitionEvent.recognitionStillActive, aidlEvent.halEventReceivedMillis);

    }



    public static SoundTrigger.RecognitionEvent aidl2apiPhraseRecognitionEvent(

            int modelHandle, int captureSession,

            PhraseRecognitionEvent aidlEvent) {

            int modelHandle, int captureSession, PhraseRecognitionEventSys aidlEvent) {

        PhraseRecognitionEvent recognitionEvent = aidlEvent.phraseRecognitionEvent;

        SoundTrigger.KeyphraseRecognitionExtra[] apiExtras =

                new SoundTrigger.KeyphraseRecognitionExtra[aidlEvent.phraseExtras.length];

        for (int i = 0; i < aidlEvent.phraseExtras.length; ++i) {

            apiExtras[i] = aidl2apiPhraseRecognitionExtra(aidlEvent.phraseExtras[i]);

                new SoundTrigger.KeyphraseRecognitionExtra[recognitionEvent.phraseExtras.length];

        for (int i = 0; i < recognitionEvent.phraseExtras.length; ++i) {

            apiExtras[i] = aidl2apiPhraseRecognitionExtra(recognitionEvent.phraseExtras[i]);

        }

        // The API recognition event doesn't allow for a null audio format, even though it doesn't

        // always make sense. We thus replace it with a default.

        AudioFormat audioFormat = aidl2apiAudioFormatWithDefault(aidlEvent.common.audioConfig,

        AudioFormat audioFormat = aidl2apiAudioFormatWithDefault(

                recognitionEvent.common.audioConfig,

                true /*isInput*/);

        // TODO(b/265852186) propagate a timestamp from aidl interfaces

        return new SoundTrigger.KeyphraseRecognitionEvent(aidlEvent.common.status, modelHandle,

                aidlEvent.common.captureAvailable, captureSession, aidlEvent.common.captureDelayMs,

                aidlEvent.common.capturePreambleMs, aidlEvent.common.triggerInData, audioFormat,

                aidlEvent.common.data, apiExtras, -1 /* halEventReceivedMillis */);

        return new SoundTrigger.KeyphraseRecognitionEvent(recognitionEvent.common.status,

                modelHandle,

                recognitionEvent.common.captureAvailable, captureSession,

                recognitionEvent.common.captureDelayMs,

                recognitionEvent.common.capturePreambleMs, recognitionEvent.common.triggerInData,

                audioFormat,

                recognitionEvent.common.data, apiExtras, aidlEvent.halEventReceivedMillis);

    }



    // In case of a null input returns a non-null valid output.

core/java/android/hardware/soundtrigger/SoundTriggerModule.java

+4 −4
Original line number Diff line number Diff line
@@ -22,13 +22,13 @@ import android.compat.annotation.UnsupportedAppUsage; 
import android.media.permission.ClearCallingIdentityContext;

import android.media.permission.Identity;

import android.media.permission.SafeCloseable;

import android.media.soundtrigger.PhraseRecognitionEvent;

import android.media.soundtrigger.PhraseSoundModel;

import android.media.soundtrigger.RecognitionEvent;

import android.media.soundtrigger.SoundModel;

import android.media.soundtrigger_middleware.ISoundTriggerCallback;

import android.media.soundtrigger_middleware.ISoundTriggerMiddlewareService;

import android.media.soundtrigger_middleware.ISoundTriggerModule;

import android.media.soundtrigger_middleware.PhraseRecognitionEventSys;

import android.media.soundtrigger_middleware.RecognitionEventSys;

import android.os.Build;

import android.os.Handler;

import android.os.IBinder;
@@ -398,7 +398,7 @@ public class SoundTriggerModule { 
        }



        @Override

        public synchronized void onRecognition(int handle, RecognitionEvent event,

        public synchronized void onRecognition(int handle, RecognitionEventSys event,

                int captureSession)

                throws RemoteException {

            Message m = mHandler.obtainMessage(EVENT_RECOGNITION,
@@ -407,7 +407,7 @@ public class SoundTriggerModule { 
        }



        @Override

        public synchronized void onPhraseRecognition(int handle, PhraseRecognitionEvent event,

        public synchronized void onPhraseRecognition(int handle, PhraseRecognitionEventSys event,

                int captureSession)

                throws RemoteException {

            Message m = mHandler.obtainMessage(EVENT_RECOGNITION,

media/aidl/android/media/soundtrigger_middleware/ISoundTriggerCallback.aidl

+4 −4
Original line number Diff line number Diff line
@@ -15,8 +15,8 @@ 
 */

package android.media.soundtrigger_middleware;



import android.media.soundtrigger.RecognitionEvent;

import android.media.soundtrigger.PhraseRecognitionEvent;

import android.media.soundtrigger_middleware.RecognitionEventSys;

import android.media.soundtrigger_middleware.PhraseRecognitionEventSys;



/**

 * Main interface for a client to get notifications of events coming from this module.
@@ -31,7 +31,7 @@ oneway interface ISoundTriggerCallback { 
     * In case of abortion, the caller may retry after the next onRecognitionAvailabilityChange()

     * callback.

     */

    void onRecognition(int modelHandle, in RecognitionEvent event, int captureSession);

    void onRecognition(int modelHandle, in RecognitionEventSys event, int captureSession);

     /**

      * Invoked whenever a phrase recognition event is triggered (typically, on recognition, but

      * also in case of external aborting of a recognition or a forced recognition event - see the
@@ -39,7 +39,7 @@ oneway interface ISoundTriggerCallback { 
      * In case of abortion, the caller may retry after the next onRecognitionAvailabilityChange()

      * callback.

      */

    void onPhraseRecognition(int modelHandle, in PhraseRecognitionEvent event, int captureSession);

    void onPhraseRecognition(int modelHandle, in PhraseRecognitionEventSys event, int captureSession);

    /**

     * Notifies the client that some start/load operations that have previously failed for resource

     * reasons (threw a ServiceSpecificException(RESOURCE_CONTENTION) or have been preempted) may

media/aidl/android/media/soundtrigger_middleware/PhraseRecognitionEventSys.aidl

0 → 100644
+36 −0
Original line number Diff line number Diff line 
/*

 * Copyright (C) 2023 The Android Open Source Project

 *

 * Licensed under the Apache License, Version 2.0 (the "License");

 * you may not use this file except in compliance with the License.

 * You may obtain a copy of the License at

 *

 *      http://www.apache.org/licenses/LICENSE-2.0

 *

 * Unless required by applicable law or agreed to in writing, software

 * distributed under the License is distributed on an "AS IS" BASIS,

 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

 * See the License for the specific language governing permissions and

 * limitations under the License.

 */

package android.media.soundtrigger_middleware;



import android.media.soundtrigger.PhraseRecognitionEvent;



/**

 * Wrapper to android.media.soundtrigger.RecognitionEvent providing additional fields used by the

 * framework.

 */

parcelable PhraseRecognitionEventSys {



    PhraseRecognitionEvent phraseRecognitionEvent;

    /**

     * Timestamp of when the trigger event from SoundTriggerHal was received by the

     * framework.

     *

     * <p>same units and timebase as {@link SystemClock#elapsedRealtime()}.

     * The value will be -1 if the event was not generated from the HAL.

     */

    // @ElapsedRealtimeLong

    long halEventReceivedMillis = -1;

}

media/aidl/android/media/soundtrigger_middleware/RecognitionEventSys.aidl

0 → 100644
+36 −0
Original line number Diff line number Diff line 
/*

 * Copyright (C) 2023 The Android Open Source Project

 *

 * Licensed under the Apache License, Version 2.0 (the "License");

 * you may not use this file except in compliance with the License.

 * You may obtain a copy of the License at

 *

 *      http://www.apache.org/licenses/LICENSE-2.0

 *

 * Unless required by applicable law or agreed to in writing, software

 * distributed under the License is distributed on an "AS IS" BASIS,

 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

 * See the License for the specific language governing permissions and

 * limitations under the License.

 */

package android.media.soundtrigger_middleware;



import android.media.soundtrigger.RecognitionEvent;



/**

 * Wrapper to android.media.soundtrigger.RecognitionEvent providing additional fields used by the

 * framework.

 */

parcelable RecognitionEventSys {



    RecognitionEvent recognitionEvent;

    /**

     * Timestamp of when the trigger event from SoundTriggerHal was received by the

     * framework.

     *

     * <p>same units and timebase as {@link SystemClock#elapsedRealtime()}.

     * The value will be -1 if the event was not generated from the HAL.

     */

    // @ElapsedRealtimeLong

    long halEventReceivedMillis = -1;

}