Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit b3f315a7 authored by Eric Laurent's avatar Eric Laurent
Browse files

audioflinger: add virtualizer stage mixer thread 

Add specialized type of MixerThread for when
a global virtualizer stage with headtracking is used on a
HAL output stream.
This mixer thread has the following requirements:
- Implement a multichannel mixer but a stereo output stream
- Have an output stage effect chain with either a virtualizer stage
effect or plain downmixer effect and possbily other post processing effects
placed after the virtualizer/downmixer
- Being able to handle relatively short audio HAL buffers (5 to 10 ms)
while allowing audio effects to be applied (not a FastMixer).

Bug: 188502620
Test: make

Change-Id: I4dd5b4cd7da7ae0d6562a9a319bdb06b2e31e4e4
parent f1f22e77

services/audioflinger/AudioFlinger.cpp

+10 −2
Original line number Diff line number Diff line
@@ -2569,7 +2569,14 @@ sp<AudioFlinger::ThreadBase> AudioFlinger::openOutput_l(audio_module_handle_t mo 
            return thread;

        } else {

            sp<PlaybackThread> thread;

            if (flags & AUDIO_OUTPUT_FLAG_COMPRESS_OFFLOAD) {

            //TODO: b/193496180 use virtualizer stage flag at audio HAL when available

            if (flags == (audio_output_flags_t)(AUDIO_OUTPUT_FLAG_FAST

                                                    | AUDIO_OUTPUT_FLAG_DEEP_BUFFER)) {

                thread = new VirtualizerStageThread(this, outputStream, *output,

                                                    mSystemReady, mixerConfig);

                ALOGD("openOutput_l() created virtualizer output: ID %d thread %p",

                      *output, thread.get());

            } else if (flags & AUDIO_OUTPUT_FLAG_COMPRESS_OFFLOAD) {

                thread = new OffloadThread(this, outputStream, *output, mSystemReady);

                ALOGV("openOutput_l() created offload output: ID %d thread %p",

                      *output, thread.get());
@@ -3806,7 +3813,8 @@ status_t AudioFlinger::createEffect(const media::CreateEffectRequest& request, 
                io = mPlaybackThreads.keyAt(0);

            }

            ALOGV("createEffect() got io %d for effect %s", io, descOut.name);

        } else if (checkPlaybackThread_l(io) != nullptr) {

        } else if (checkPlaybackThread_l(io) != nullptr

                        && sessionId != AUDIO_SESSION_OUTPUT_STAGE) {

            // allow only one effect chain per sessionId on mPlaybackThreads.

            for (size_t i = 0; i < mPlaybackThreads.size(); i++) {

                const audio_io_handle_t checkIo = mPlaybackThreads.keyAt(i);

services/audioflinger/Threads.cpp

+120 −10
Original line number Diff line number Diff line
@@ -50,8 +50,10 @@ 
#include <audio_utils/format.h>

#include <audio_utils/minifloat.h>

#include <audio_utils/safe_math.h>

#include <system/audio_effects/effect_ns.h>

#include <system/audio_effects/effect_aec.h>

#include <system/audio_effects/effect_downmix.h>

#include <system/audio_effects/effect_ns.h>

#include <system/audio_effects/effect_virtualizer_stage.h>

#include <system/audio.h>



// NBAIO implementations
@@ -507,6 +509,8 @@ const char *AudioFlinger::ThreadBase::threadTypeToString(AudioFlinger::ThreadBas 
        return "MMAP_PLAYBACK";

    case MMAP_CAPTURE:

        return "MMAP_CAPTURE";

    case VIRTUALIZER_STAGE:

        return "VIRTUALIZER_STAGE";

    default:

        return "unknown";

    }
@@ -722,6 +726,19 @@ void AudioFlinger::ThreadBase::sendResizeBufferConfigEvent_l(int32_t maxSharedAu 
    sendConfigEvent_l(configEvent);

}



void AudioFlinger::ThreadBase::sendCheckOutputStageEffectsEvent()

{

    Mutex::Autolock _l(mLock);

    sendCheckOutputStageEffectsEvent_l();

}



void AudioFlinger::ThreadBase::sendCheckOutputStageEffectsEvent_l()

{

    sp<ConfigEvent> configEvent =

            (ConfigEvent *)new CheckOutputStageEffectsEvent();

    sendConfigEvent_l(configEvent);

}



// post condition: mConfigEvents.isEmpty()

void AudioFlinger::ThreadBase::processConfigEvents_l()

{
@@ -784,6 +801,11 @@ void AudioFlinger::ThreadBase::processConfigEvents_l() 
                    (ResizeBufferConfigEventData *)event->mData.get();

            resizeInputBuffer_l(data->mMaxSharedAudioHistoryMs);

        } break;



        case CFG_EVENT_CHECK_OUTPUT_STAGE_EFFECTS: {

            setCheckOutputStageEffects();

        } break;



        default:

            ALOG_ASSERT(false, "processConfigEvents_l() unknown event type %d", event->mType);

            break;
@@ -1008,6 +1030,8 @@ String16 AudioFlinger::ThreadBase::getWakeLockTag() 
        return String16("MmapPlayback");

    case MMAP_CAPTURE:

        return String16("MmapCapture");

    case VIRTUALIZER_STAGE:

        return String16("AudioVirt");

    default:

        ALOG_ASSERT(false);

        return String16("AudioUnknown");
@@ -1401,6 +1425,13 @@ status_t AudioFlinger::PlaybackThread::checkEffectCompatibility_l( 
            return BAD_VALUE;

        }

        break;

    case VIRTUALIZER_STAGE:

        if (!audio_is_global_session(sessionId)) {

            ALOGW("checkEffectCompatibility_l(): non global effect %s on VIRTUALIZER_STAGE"

                    " thread %s", desc->name, mThreadName);

            return BAD_VALUE;

        }

        break;

    default:

        LOG_ALWAYS_FATAL("checkEffectCompatibility_l(): wrong thread type %d", mType);

    }
@@ -1489,6 +1520,7 @@ sp<AudioFlinger::EffectHandle> AudioFlinger::ThreadBase::createEffect_l( 
        lStatus = handle->initCheck();

        if (lStatus == OK) {

            lStatus = effect->addHandle(handle.get());

            sendCheckOutputStageEffectsEvent_l();

        }

        if (enabled != NULL) {

            *enabled = (int)effect->isEnabled();
@@ -1531,6 +1563,7 @@ void AudioFlinger::ThreadBase::disconnectEffectHandle(EffectHandle *handle, 
        if (remove) {

            removeEffect_l(effect, true);

        }

        sendCheckOutputStageEffectsEvent_l();

    }

    if (remove) {

        mAudioFlinger->updateOrphanEffectChains(effect);
@@ -1908,12 +1941,12 @@ AudioFlinger::PlaybackThread::PlaybackThread(const sp<AudioFlinger>& audioFlinge 
                                             audio_config_base_t *mixerConfig)

    :   ThreadBase(audioFlinger, id, type, systemReady, true /* isOut */),

        mNormalFrameCount(0), mSinkBuffer(NULL),

        mMixerBufferEnabled(AudioFlinger::kEnableExtendedPrecision),

        mMixerBufferEnabled(AudioFlinger::kEnableExtendedPrecision || type == VIRTUALIZER_STAGE),

        mMixerBuffer(NULL),

        mMixerBufferSize(0),

        mMixerBufferFormat(AUDIO_FORMAT_INVALID),

        mMixerBufferValid(false),

        mEffectBufferEnabled(AudioFlinger::kEnableExtendedPrecision),

        mEffectBufferEnabled(AudioFlinger::kEnableExtendedPrecision || type == VIRTUALIZER_STAGE),

        mEffectBuffer(NULL),

        mEffectBufferSize(0),

        mEffectBufferFormat(AUDIO_FORMAT_INVALID),
@@ -1971,6 +2004,12 @@ AudioFlinger::PlaybackThread::PlaybackThread(const sp<AudioFlinger>& audioFlinge 


    readOutputParameters_l();



    if (mType != VIRTUALIZER_STAGE

            && mMixerChannelMask != mChannelMask) {

        LOG_ALWAYS_FATAL("HAL channel mask %#x does not match mixer channel mask %#x",

                mChannelMask, mMixerChannelMask);

    }



    // TODO: We may also match on address as well as device type for

    // AUDIO_DEVICE_OUT_BUS, AUDIO_DEVICE_OUT_ALL_A2DP, AUDIO_DEVICE_OUT_REMOTE_SUBMIX

    if (type == MIXER || type == DIRECT || type == OFFLOAD) {
@@ -2840,8 +2879,7 @@ void AudioFlinger::PlaybackThread::readOutputParameters_l() 
    if (!audio_is_output_channel(mChannelMask)) {

        LOG_ALWAYS_FATAL("HAL channel mask %#x not valid for output", mChannelMask);

    }

    if ((mType == MIXER || mType == DUPLICATING)

            && !isValidPcmSinkChannelMask(mChannelMask)) {

    if (hasMixer() && !isValidPcmSinkChannelMask(mChannelMask)) {

        LOG_ALWAYS_FATAL("HAL channel mask %#x not supported for mixed output",

                mChannelMask);

    }
@@ -2864,8 +2902,7 @@ void AudioFlinger::PlaybackThread::readOutputParameters_l() 
    if (!audio_is_valid_format(mFormat)) {

        LOG_ALWAYS_FATAL("HAL format %#x not valid for output", mFormat);

    }

    if ((mType == MIXER || mType == DUPLICATING)

            && !isValidPcmSinkFormat(mFormat)) {

    if (hasMixer() && !isValidPcmSinkFormat(mFormat)) {

        LOG_FATAL("HAL format %#x not supported for mixed output",

                mFormat);

    }
@@ -2874,7 +2911,7 @@ void AudioFlinger::PlaybackThread::readOutputParameters_l() 
    LOG_ALWAYS_FATAL_IF(result != OK,

            "Error when retrieving output stream buffer size: %d", result);

    mFrameCount = mBufferSize / mFrameSize;

    if ((mType == MIXER || mType == DUPLICATING) && (mFrameCount & 15)) {

    if (hasMixer() && (mFrameCount & 15)) {

        ALOGW("HAL output buffer size is %zu frames but AudioMixer requires multiples of 16 frames",

                mFrameCount);

    }
@@ -2947,7 +2984,7 @@ void AudioFlinger::PlaybackThread::readOutputParameters_l() 
    }

    mNormalFrameCount = multiplier * mFrameCount;

    // round up to nearest 16 frames to satisfy AudioMixer

    if (mType == MIXER || mType == DUPLICATING) {

    if (hasMixer()) {

        mNormalFrameCount = (mNormalFrameCount + 15) & ~15;

    }

    ALOGI("HAL output buffer size %zu frames, normal sink buffer size %zu frames", mFrameCount,
@@ -3566,6 +3603,8 @@ bool AudioFlinger::PlaybackThread::threadLoop() 


    audio_patch_handle_t lastDownstreamPatchHandle = AUDIO_PATCH_HANDLE_NONE;



    sendCheckOutputStageEffectsEvent();



    // loopCount is used for statistics and diagnostics.

    for (int64_t loopCount = 0; !exitPending(); ++loopCount)

    {
@@ -3622,11 +3661,18 @@ bool AudioFlinger::PlaybackThread::threadLoop() 
            }

        }



        if (mCheckOutputStageEffects.exchange(false)) {

            checkOutputStageEffects();

        }



        { // scope for mLock



            Mutex::Autolock _l(mLock);



            processConfigEvents_l();

            if (mCheckOutputStageEffects.load()) {

                continue;

            }



            // See comment at declaration of logString for why this is done under mLock

            if (logString != NULL) {
@@ -5618,7 +5664,8 @@ AudioFlinger::PlaybackThread::mixer_state AudioFlinger::MixerThread::prepareTrac 
    // remove all the tracks that need to be...

    removeTracks_l(*tracksToRemove);



    if (getEffectChain_l(AUDIO_SESSION_OUTPUT_MIX) != 0) {

    if (getEffectChain_l(AUDIO_SESSION_OUTPUT_MIX) != 0 ||

            getEffectChain_l(AUDIO_SESSION_OUTPUT_STAGE) != 0) {

        mEffectBufferValid = true;

    }
@@ -7014,6 +7061,69 @@ void AudioFlinger::DuplicatingThread::cacheParameters_l() 
    MixerThread::cacheParameters_l();

}



// ----------------------------------------------------------------------------



AudioFlinger::VirtualizerStageThread::VirtualizerStageThread(const sp<AudioFlinger>& audioFlinger,

                                                             AudioStreamOut* output,

                                                             audio_io_handle_t id,

                                                             bool systemReady,

                                                             audio_config_base_t *mixerConfig)

    : MixerThread(audioFlinger, output, id, systemReady, VIRTUALIZER_STAGE, mixerConfig)

{

}



void AudioFlinger::VirtualizerStageThread::checkOutputStageEffects()

{

    bool hasVirtualizer = false;

    bool hasDownMixer = false;

    sp<EffectHandle> finalDownMixer;

    {

        Mutex::Autolock _l(mLock);

        sp<EffectChain> chain = getEffectChain_l(AUDIO_SESSION_OUTPUT_STAGE);

        if (chain != 0) {

            hasVirtualizer = chain->getEffectFromType_l(FX_IID_VIRTUALIZER_STAGE) != nullptr;

            hasDownMixer = chain->getEffectFromType_l(EFFECT_UIID_DOWNMIX) != nullptr;

        }



        finalDownMixer = mFinalDownMixer;

        mFinalDownMixer.clear();

    }



    if (hasVirtualizer) {

        if (finalDownMixer != nullptr) {

            int32_t ret;

            finalDownMixer->disable(&ret);

        }

        finalDownMixer.clear();

    } else if (!hasDownMixer) {

        std::vector<effect_descriptor_t> descriptors;

        status_t status = mAudioFlinger->mEffectsFactoryHal->getDescriptors(

                                                        EFFECT_UIID_DOWNMIX, &descriptors);

        if (status != NO_ERROR) {

            return;

        }

        ALOG_ASSERT(!descriptors.empty(),

                "%s getDescriptors() returned no error but empty list", __func__);



        finalDownMixer = createEffect_l(nullptr /*client*/, nullptr /*effectClient*/,

                0 /*priority*/, AUDIO_SESSION_OUTPUT_STAGE, &descriptors[0], nullptr /*enabled*/,

                &status, false /*pinned*/, false /*probe*/);



        if (finalDownMixer == nullptr || (status != NO_ERROR && status != ALREADY_EXISTS)) {

            ALOGW("%s error creating downmixer %d", __func__, status);

            finalDownMixer.clear();

        } else {

            int32_t ret;

            finalDownMixer->enable(&ret);

        }

    }



    {

        Mutex::Autolock _l(mLock);

        mFinalDownMixer = finalDownMixer;

    }

}





// ----------------------------------------------------------------------------

//      Record

services/audioflinger/Threads.h

+59 −9
Original line number Diff line number Diff line
@@ -32,6 +32,7 @@ public: 
        OFFLOAD,            // Thread class is OffloadThread

        MMAP_PLAYBACK,      // Thread class for MMAP playback stream

        MMAP_CAPTURE,       // Thread class for MMAP capture stream

        VIRTUALIZER_STAGE,  //

        // If you add any values here, also update ThreadBase::threadTypeToString()

    };
@@ -53,7 +54,8 @@ public: 
        CFG_EVENT_CREATE_AUDIO_PATCH,

        CFG_EVENT_RELEASE_AUDIO_PATCH,

        CFG_EVENT_UPDATE_OUT_DEVICE,

        CFG_EVENT_RESIZE_BUFFER

        CFG_EVENT_RESIZE_BUFFER,

        CFG_EVENT_CHECK_OUTPUT_STAGE_EFFECTS

    };



    class ConfigEventData: public RefBase {
@@ -87,7 +89,13 @@ public: 
    public:

        virtual ~ConfigEvent() {}



        void dump(char *buffer, size_t size) { mData->dump(buffer, size); }

        void dump(char *buffer, size_t size) {

            snprintf(buffer, size, "Event type: %d\n", mType);

            if (mData != nullptr) {

                snprintf(buffer, size, "Data:\n");

                mData->dump(buffer, size);

            }

        }



        const int mType; // event type e.g. CFG_EVENT_IO

        Mutex mLock;     // mutex associated with mCond
@@ -110,7 +118,7 @@ public: 
            mEvent(event), mPid(pid), mPortId(portId) {}



        virtual  void dump(char *buffer, size_t size) {

            snprintf(buffer, size, "IO event: event %d\n", mEvent);

            snprintf(buffer, size, "- IO event: event %d\n", mEvent);

        }



        const audio_io_config_event mEvent;
@@ -133,7 +141,7 @@ public: 
            mPid(pid), mTid(tid), mPrio(prio), mForApp(forApp) {}



        virtual  void dump(char *buffer, size_t size) {

            snprintf(buffer, size, "Prio event: pid %d, tid %d, prio %d, for app? %d\n",

            snprintf(buffer, size, "- Prio event: pid %d, tid %d, prio %d, for app? %d\n",

                    mPid, mTid, mPrio, mForApp);

        }
@@ -158,7 +166,7 @@ public: 
            mKeyValuePairs(keyValuePairs) {}



        virtual  void dump(char *buffer, size_t size) {

            snprintf(buffer, size, "KeyValue: %s\n", mKeyValuePairs.string());

            snprintf(buffer, size, "- KeyValue: %s\n", mKeyValuePairs.string());

        }



        const String8 mKeyValuePairs;
@@ -181,7 +189,7 @@ public: 
            mPatch(patch), mHandle(handle) {}



        virtual  void dump(char *buffer, size_t size) {

            snprintf(buffer, size, "Patch handle: %u\n", mHandle);

            snprintf(buffer, size, "- Patch handle: %u\n", mHandle);

        }



        const struct audio_patch mPatch;
@@ -205,7 +213,7 @@ public: 
            mHandle(handle) {}



        virtual  void dump(char *buffer, size_t size) {

            snprintf(buffer, size, "Patch handle: %u\n", mHandle);

            snprintf(buffer, size, "- Patch handle: %u\n", mHandle);

        }



        audio_patch_handle_t mHandle;
@@ -227,7 +235,7 @@ public: 
            mOutDevices(outDevices) {}



        virtual void dump(char *buffer, size_t size) {

            snprintf(buffer, size, "Devices: %s", android::toString(mOutDevices).c_str());

            snprintf(buffer, size, "- Devices: %s", android::toString(mOutDevices).c_str());

        }



        DeviceDescriptorBaseVector mOutDevices;
@@ -249,7 +257,7 @@ public: 
            mMaxSharedAudioHistoryMs(maxSharedAudioHistoryMs) {}



        virtual void dump(char *buffer, size_t size) {

            snprintf(buffer, size, "mMaxSharedAudioHistoryMs: %d", mMaxSharedAudioHistoryMs);

            snprintf(buffer, size, "- mMaxSharedAudioHistoryMs: %d", mMaxSharedAudioHistoryMs);

        }



        int32_t mMaxSharedAudioHistoryMs;
@@ -265,6 +273,16 @@ public: 
        virtual ~ResizeBufferConfigEvent() {}

    };



    class CheckOutputStageEffectsEvent : public ConfigEvent {

    public:

        CheckOutputStageEffectsEvent() :

            ConfigEvent(CFG_EVENT_CHECK_OUTPUT_STAGE_EFFECTS) {

        }



        virtual ~CheckOutputStageEffectsEvent() {}

    };





    class PMDeathRecipient : public IBinder::DeathRecipient {

    public:

        explicit    PMDeathRecipient(const wp<ThreadBase>& thread) : mThread(thread) {}
@@ -333,7 +351,11 @@ public: 
                status_t    sendUpdateOutDeviceConfigEvent(

                                    const DeviceDescriptorBaseVector& outDevices);

                void        sendResizeBufferConfigEvent_l(int32_t maxSharedAudioHistoryMs);

                void        sendCheckOutputStageEffectsEvent();

                void        sendCheckOutputStageEffectsEvent_l();



                void        processConfigEvents_l();

    virtual     void        setCheckOutputStageEffects() {}

    virtual     void        cacheParameters_l() = 0;

    virtual     status_t    createAudioPatch_l(const struct audio_patch *patch,

                                               audio_patch_handle_t *handle) = 0;
@@ -887,6 +909,8 @@ protected: 
                                mActiveTracks.updatePowerState(this, true /* force */);

                            }



    virtual     void        checkOutputStageEffects() {}



                void        dumpInternals_l(int fd, const Vector<String16>& args) override;

                void        dumpTracks_l(int fd, const Vector<String16>& args) override;
@@ -1025,6 +1049,9 @@ public: 


                PlaybackThread::Track* getTrackById_l(audio_port_handle_t trackId);



                bool hasMixer() const {

                    return mType == MIXER || mType == DUPLICATING || mType == VIRTUALIZER_STAGE;

                }

protected:

    // updated by readOutputParameters_l()

    size_t                          mNormalFrameCount;  // normal mixer and effects
@@ -1147,6 +1174,9 @@ protected: 


    // Cache various calculated values, at threadLoop() entry and after a parameter change

    virtual     void        cacheParameters_l();

                void        setCheckOutputStageEffects() override {

                                mCheckOutputStageEffects.store(true);

                            }



    virtual     uint32_t    correctLatency_l(uint32_t latency) const;
@@ -1327,6 +1357,8 @@ protected: 
                // audio patch used by the downstream software patch.

                // Only used if ThreadBase::mIsMsdDevice is true.

                struct audio_patch mDownStreamPatch;



                std::atomic_bool mCheckOutputStageEffects{};

};



class MixerThread : public PlaybackThread {
@@ -1625,6 +1657,24 @@ public: 
    }

};



class VirtualizerStageThread : public MixerThread {

public:

    VirtualizerStageThread(const sp<AudioFlinger>& audioFlinger,

                           AudioStreamOut* output,

                           audio_io_handle_t id,

                           bool systemReady,

                           audio_config_base_t *mixerConfig);

            ~VirtualizerStageThread() override {}



            bool hasFastMixer() const override { return false; }



protected:

            void checkOutputStageEffects() override;



private:

            sp<EffectHandle> mFinalDownMixer;

};



// record thread

class RecordThread : public ThreadBase

{