Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 7d5b2623 authored by Jean-Michel Trivi's avatar Jean-Michel Trivi
Browse files

AudioMixer uses downmix effect for multichannel content 

In the AudioMixer structure associated with each track, add an object
  that acts as the buffer provider when the track has more than two
  channels of input in the mixer. This object, DownmixerBufferProvider,
  gets audio from the actual buffer provider of the track, and applies
  a downmix effect on it.
The downmix effect is created and configured when the track gets
  created in AudioFlinger, which causes AudioMixer::getTrackName()
  to be called with the new track's channel mask. It is released
  when the track is disabled in the mixer.

Change-Id: I05281ed5f61bef663a8af7ca7d5ceac3517c82db
parent 4b75a9c8

media/libeffects/data/audio_effects.conf

+7 −0
Original line number Diff line number Diff line
@@ -18,6 +18,9 @@ libraries { 
  pre_processing {

    path /system/lib/soundfx/libaudiopreprocessing.so

  }

  downmix {

    path /system/lib/soundfx/libdownmix.so

  }

}



# list of effects to load. Each effect element must contain a "library" and a "uuid" element.
@@ -72,6 +75,10 @@ effects { 
    library visualizer

    uuid d069d9e0-8329-11df-9168-0002a5d5c51b

  }

  downmix {

    library downmix

    uuid 93f04452-e4fe-41cc-91f9-e475b6d1d69f

  }

  agc {

    library pre_processing

    uuid aa8130e0-66fc-11e0-bad0-0002a5d5c51b

media/libeffects/downmix/EffectDownmix.c

+1 −1
Original line number Diff line number Diff line
@@ -610,7 +610,7 @@ int Downmix_setParameter(downmix_object_t *pDownmixer, int32_t param, size_t siz 
        }

        value16 = *(int16_t *)pValue;

        ALOGV("set DOWNMIX_PARAM_TYPE, type %d", value16);

        if (!((value16 > DOWNMIX_TYPE_INVALID) && (value16 < DOWNMIX_TYPE_LAST))) {

        if (!((value16 > DOWNMIX_TYPE_INVALID) && (value16 <= DOWNMIX_TYPE_LAST))) {

            ALOGE("Downmix_setParameter invalid DOWNMIX_PARAM_TYPE value %d", value16);

            return -EINVAL;

        } else {

services/audioflinger/AudioFlinger.cpp

+11 −5
Original line number Diff line number Diff line
@@ -2639,9 +2639,15 @@ void AudioFlinger::MixerThread::invalidateTracks(audio_stream_type_t streamType) 
}



// getTrackName_l() must be called with ThreadBase::mLock held

int AudioFlinger::MixerThread::getTrackName_l()

int AudioFlinger::MixerThread::getTrackName_l(audio_channel_mask_t channelMask)

{

    return mAudioMixer->getTrackName();

    int name = mAudioMixer->getTrackName();

    if (name >= 0) {

        mAudioMixer->setParameter(name,

                AudioMixer::TRACK,

                AudioMixer::CHANNEL_MASK, (void *)channelMask);

    }

    return name;

}



// deleteTrackName_l() must be called with ThreadBase::mLock held
@@ -2738,7 +2744,7 @@ bool AudioFlinger::MixerThread::checkForNewParameters_l() 
                readOutputParameters();

                mAudioMixer = new AudioMixer(mFrameCount, mSampleRate);

                for (size_t i = 0; i < mTracks.size() ; i++) {

                    int name = getTrackName_l();

                    int name = getTrackName_l((audio_channel_mask_t)mTracks[i]->mChannelMask);

                    if (name < 0) break;

                    mTracks[i]->mName = name;

                    // limit track sample rate to 2 x new output sample rate
@@ -3066,7 +3072,7 @@ void AudioFlinger::DirectOutputThread::threadLoop_sleepTime() 
}



// getTrackName_l() must be called with ThreadBase::mLock held

int AudioFlinger::DirectOutputThread::getTrackName_l()

int AudioFlinger::DirectOutputThread::getTrackName_l(audio_channel_mask_t channelMask)

{

    return 0;

}
@@ -3528,7 +3534,7 @@ AudioFlinger::PlaybackThread::Track::Track( 
        // 16 bit because data is converted to 16 bit before being stored in buffer by AudioTrack

        mCblk->frameSize = audio_is_linear_pcm(format) ? mChannelCount * sizeof(int16_t) : sizeof(uint8_t);

        // to avoid leaking a track name, do not allocate one unless there is an mCblk

        mName = thread->getTrackName_l();

        mName = thread->getTrackName_l((audio_channel_mask_t)channelMask);

        if (mName < 0) {

            ALOGE("no more track names available");

        }

services/audioflinger/AudioFlinger.h

+5 −4
Original line number Diff line number Diff line
@@ -964,8 +964,9 @@ public: 
    protected:

        SortedVector< wp<Track> >       mActiveTracks;



        // Allocate a track name.  Returns name >= 0 if successful, -1 on failure.

        virtual int             getTrackName_l() = 0;

        // Allocate a track name for a given channel mask.

        //   Returns name >= 0 if successful, -1 on failure.

        virtual int             getTrackName_l(audio_channel_mask_t channelMask) = 0;

        virtual void            deleteTrackName_l(int name) = 0;



        // Time to sleep between cycles when:
@@ -1057,7 +1058,7 @@ public: 


    protected:

        virtual     mixer_state prepareTracks_l(Vector< sp<Track> > *tracksToRemove);

        virtual     int         getTrackName_l();

        virtual     int         getTrackName_l(audio_channel_mask_t channelMask);

        virtual     void        deleteTrackName_l(int name);

        virtual     uint32_t    idleSleepTimeUs() const;

        virtual     uint32_t    suspendSleepTimeUs() const;
@@ -1082,7 +1083,7 @@ public: 
        virtual     bool        checkForNewParameters_l();



    protected:

        virtual     int         getTrackName_l();

        virtual     int         getTrackName_l(audio_channel_mask_t channelMask);

        virtual     void        deleteTrackName_l(int name);

        virtual     uint32_t    activeSleepTimeUs() const;

        virtual     uint32_t    idleSleepTimeUs() const;

services/audioflinger/AudioMixer.cpp

+217 −4
Original line number Diff line number Diff line
@@ -36,11 +36,62 @@ 
#include <common_time/local_clock.h>

#include <common_time/cc_helper.h>



#include <media/EffectsFactoryApi.h>



#include "AudioMixer.h"



namespace android {



// ----------------------------------------------------------------------------

AudioMixer::DownmixerBufferProvider::DownmixerBufferProvider() : AudioBufferProvider(),

        mTrackBufferProvider(NULL), mDownmixHandle(NULL)

{

}



AudioMixer::DownmixerBufferProvider::~DownmixerBufferProvider()

{

    ALOGV("AudioMixer deleting DownmixerBufferProvider (%p)", this);

    EffectRelease(mDownmixHandle);

}



status_t AudioMixer::DownmixerBufferProvider::getNextBuffer(AudioBufferProvider::Buffer *pBuffer,

        int64_t pts) {

    //ALOGV("DownmixerBufferProvider::getNextBuffer()");

    if (this->mTrackBufferProvider != NULL) {

        status_t res = mTrackBufferProvider->getNextBuffer(pBuffer, pts);

        if (res == OK) {

            mDownmixConfig.inputCfg.buffer.frameCount = pBuffer->frameCount;

            mDownmixConfig.inputCfg.buffer.raw = pBuffer->raw;

            mDownmixConfig.outputCfg.buffer.frameCount = pBuffer->frameCount;

            mDownmixConfig.outputCfg.buffer.raw = mDownmixConfig.inputCfg.buffer.raw;

            // in-place so overwrite the buffer contents, has been set in prepareTrackForDownmix()

            //mDownmixConfig.outputCfg.accessMode = EFFECT_BUFFER_ACCESS_WRITE;



            res = (*mDownmixHandle)->process(mDownmixHandle,

                    &mDownmixConfig.inputCfg.buffer, &mDownmixConfig.outputCfg.buffer);

            ALOGV("getNextBuffer is downmixing");

        }

        return res;

    } else {

        ALOGE("DownmixerBufferProvider::getNextBuffer() error: NULL track buffer provider");

        return NO_INIT;

    }

}



void AudioMixer::DownmixerBufferProvider::releaseBuffer(AudioBufferProvider::Buffer *pBuffer) {

    ALOGV("DownmixerBufferProvider::releaseBuffer()");

    if (this->mTrackBufferProvider != NULL) {

        mTrackBufferProvider->releaseBuffer(pBuffer);

    } else {

        ALOGE("DownmixerBufferProvider::releaseBuffer() error: NULL track buffer provider");

    }

}





// ----------------------------------------------------------------------------

bool AudioMixer::isMultichannelCapable = false;



effect_descriptor_t AudioMixer::dwnmFxDesc;



AudioMixer::AudioMixer(size_t frameCount, uint32_t sampleRate, uint32_t maxNumTracks)

    :   mTrackNames(0), mConfiguredNames((1 << maxNumTracks) - 1), mSampleRate(sampleRate)
@@ -70,6 +121,28 @@ AudioMixer::AudioMixer(size_t frameCount, uint32_t sampleRate, uint32_t maxNumTr 
        t->localTimeFreq = lc.getLocalFreq();

        t++;

    }



    // find multichannel downmix effect if we have to play multichannel content

    uint32_t numEffects = 0;

    int ret = EffectQueryNumberEffects(&numEffects);

    if (ret != 0) {

        ALOGE("AudioMixer() error %d querying number of effects", ret);

        return;

    }

    ALOGV("EffectQueryNumberEffects() numEffects=%d", numEffects);



    for (uint32_t i = 0 ; i < numEffects ; i++) {

        if (EffectQueryEffect(i, &dwnmFxDesc) == 0) {

            ALOGV("effect %d is called %s", i, dwnmFxDesc.name);

            if (memcmp(&dwnmFxDesc.type, EFFECT_UIID_DOWNMIX, sizeof(effect_uuid_t)) == 0) {

                ALOGI("found effect \"%s\" from %s",

                        dwnmFxDesc.name, dwnmFxDesc.implementor);

                isMultichannelCapable = true;

                break;

            }

        }

    }

    ALOGE_IF(!isMultichannelCapable, "unable to find downmix effect");

}



AudioMixer::~AudioMixer()
@@ -111,6 +184,7 @@ int AudioMixer::getTrackName() 
        t->channelMask = AUDIO_CHANNEL_OUT_STEREO;

        // setBufferProvider(name, AudioBufferProvider *) is required before enable(name)

        t->bufferProvider = NULL;

        t->downmixerBufferProvider = NULL;

        t->buffer.raw = NULL;

        // no initialization needed

        // t->buffer.frameCount
@@ -135,6 +209,113 @@ void AudioMixer::invalidateState(uint32_t mask) 
    }

 }



status_t AudioMixer::prepareTrackForDownmix(track_t* pTrack, int trackName)

{

    ALOGV("AudioMixer::prepareTrackForDownmix(%d) with mask 0x%x", trackName, pTrack->channelMask);



    if (pTrack->downmixerBufferProvider != NULL) {

        // this track had previously been configured with a downmixer, reset it

        ALOGV("AudioMixer::prepareTrackForDownmix(%d) deleting old downmixer", trackName);

        pTrack->bufferProvider = pTrack->downmixerBufferProvider->mTrackBufferProvider;

        delete pTrack->downmixerBufferProvider;

    }



    DownmixerBufferProvider* pDbp = new DownmixerBufferProvider();

    int32_t status;



    if (!isMultichannelCapable) {

        ALOGE("prepareTrackForDownmix(%d) fails: mixer doesn't support multichannel content",

                trackName);

        goto noDownmixForActiveTrack;

    }



    if (EffectCreate(&dwnmFxDesc.uuid,

            -2 /*sessionId*/, -2 /*ioId*/,// both not relevant here, using random value

            &pDbp->mDownmixHandle/*pHandle*/) != 0) {

        ALOGE("prepareTrackForDownmix(%d) fails: error creating downmixer effect", trackName);

        goto noDownmixForActiveTrack;

    }



    // channel input configuration will be overridden per-track

    pDbp->mDownmixConfig.inputCfg.channels = pTrack->channelMask;

    pDbp->mDownmixConfig.outputCfg.channels = AUDIO_CHANNEL_OUT_STEREO;

    pDbp->mDownmixConfig.inputCfg.format = AUDIO_FORMAT_PCM_16_BIT;

    pDbp->mDownmixConfig.outputCfg.format = AUDIO_FORMAT_PCM_16_BIT;

    pDbp->mDownmixConfig.inputCfg.samplingRate = pTrack->sampleRate;

    pDbp->mDownmixConfig.outputCfg.samplingRate = pTrack->sampleRate;

    pDbp->mDownmixConfig.inputCfg.accessMode = EFFECT_BUFFER_ACCESS_READ;

    pDbp->mDownmixConfig.outputCfg.accessMode = EFFECT_BUFFER_ACCESS_WRITE;

    // input and output buffer provider, and frame count will not be used as the downmix effect

    // process() function is called directly (see DownmixerBufferProvider::getNextBuffer())

    pDbp->mDownmixConfig.inputCfg.mask = EFFECT_CONFIG_SMP_RATE | EFFECT_CONFIG_CHANNELS |

            EFFECT_CONFIG_FORMAT | EFFECT_CONFIG_ACC_MODE;

    pDbp->mDownmixConfig.outputCfg.mask = pDbp->mDownmixConfig.inputCfg.mask;



    {// scope for local variables that are not used in goto label "noDownmixForActiveTrack"

        int cmdStatus;

        uint32_t replySize = sizeof(int);



        // Configure and enable downmixer

        status = (*pDbp->mDownmixHandle)->command(pDbp->mDownmixHandle,

                EFFECT_CMD_SET_CONFIG /*cmdCode*/, sizeof(effect_config_t) /*cmdSize*/,

                &pDbp->mDownmixConfig /*pCmdData*/,

                &replySize /*replySize*/, &cmdStatus /*pReplyData*/);

        if ((status != 0) || (cmdStatus != 0)) {

            ALOGE("error %d while configuring downmixer for track %d", status, trackName);

            goto noDownmixForActiveTrack;

        }

        replySize = sizeof(int);

        status = (*pDbp->mDownmixHandle)->command(pDbp->mDownmixHandle,

                EFFECT_CMD_ENABLE /*cmdCode*/, 0 /*cmdSize*/, NULL /*pCmdData*/,

                &replySize /*replySize*/, &cmdStatus /*pReplyData*/);

        if ((status != 0) || (cmdStatus != 0)) {

            ALOGE("error %d while enabling downmixer for track %d", status, trackName);

            goto noDownmixForActiveTrack;

        }



        // Set downmix type

        // parameter size rounded for padding on 32bit boundary

        const int psizePadded = ((sizeof(downmix_params_t) - 1)/sizeof(int) + 1) * sizeof(int);

        const int downmixParamSize =

                sizeof(effect_param_t) + psizePadded + sizeof(downmix_type_t);

        effect_param_t * const param = (effect_param_t *) malloc(downmixParamSize);

        param->psize = sizeof(downmix_params_t);

        const downmix_params_t downmixParam = DOWNMIX_PARAM_TYPE;

        memcpy(param->data, &downmixParam, param->psize);

        const downmix_type_t downmixType = DOWNMIX_TYPE_FOLD;

        param->vsize = sizeof(downmix_type_t);

        memcpy(param->data + psizePadded, &downmixType, param->vsize);



        status = (*pDbp->mDownmixHandle)->command(pDbp->mDownmixHandle,

                EFFECT_CMD_SET_PARAM /* cmdCode */, downmixParamSize/* cmdSize */,

                param /*pCmndData*/, &replySize /*replySize*/, &cmdStatus /*pReplyData*/);



        free(param);



        if ((status != 0) || (cmdStatus != 0)) {

            ALOGE("error %d while setting downmix type for track %d", status, trackName);

            goto noDownmixForActiveTrack;

        } else {

            ALOGV("downmix type set to %d for track %d", (int) downmixType, trackName);

        }

    }// end of scope for local variables that are not used in goto label "noDownmixForActiveTrack"



    // initialization successful:

    // - keep track of the real buffer provider in case it was set before

    pDbp->mTrackBufferProvider = pTrack->bufferProvider;

    // - we'll use the downmix effect integrated inside this

    //    track's buffer provider, and we'll use it as the track's buffer provider

    pTrack->downmixerBufferProvider = pDbp;

    pTrack->bufferProvider = pDbp;



    return NO_ERROR;



noDownmixForActiveTrack:

    delete pDbp;

    pTrack->downmixerBufferProvider = NULL;

    return NO_INIT;

}



void AudioMixer::deleteTrackName(int name)

{

    name -= TRACK0;
@@ -177,6 +358,11 @@ void AudioMixer::disable(int name) 
    track_t& track = mState.tracks[name];



    if (track.enabled) {

        if (track.downmixerBufferProvider != NULL) {

            ALOGV("AudioMixer::disable(%d) deleting downmixerBufferProvider", name);

            delete track.downmixerBufferProvider;

            track.downmixerBufferProvider = NULL;

        }

        track.enabled = false;

        ALOGV("disable(%d)", name);

        invalidateState(1 << name);
@@ -200,10 +386,14 @@ void AudioMixer::setParameter(int name, int target, int param, void *value) 
            uint32_t mask = (uint32_t)value;

            if (track.channelMask != mask) {

                uint32_t channelCount = popcount(mask);

                ALOG_ASSERT((channelCount <= MAX_NUM_CHANNELS) && (channelCount),

                        "bad channel count %u", channelCount);

                ALOG_ASSERT((channelCount <= MAX_NUM_CHANNELS_TO_DOWNMIX) && channelCount);

                track.channelMask = mask;

                track.channelCount = channelCount;

                if (channelCount > MAX_NUM_CHANNELS) {

                    ALOGV("AudioMixer::setParameter(TRACK, CHANNEL_MASK, mask=0x%x count=%d)",

                            mask, channelCount);

                    status_t status = prepareTrackForDownmix(&mState.tracks[name], name);

                }

                ALOGV("setParameter(TRACK, CHANNEL_MASK, %x)", mask);

                invalidateState(1 << name);

            }
@@ -225,6 +415,10 @@ void AudioMixer::setParameter(int name, int target, int param, void *value) 
        case FORMAT:

            ALOG_ASSERT(valueInt == AUDIO_FORMAT_PCM_16_BIT);

            break;

        // FIXME do we want to support setting the downmix type from AudioFlinger?

        //         for a specific track? or per mixer?

        /* case DOWNMIX_TYPE:

            break          */

        default:

            LOG_FATAL("bad param");

        }
@@ -350,8 +544,23 @@ void AudioMixer::setBufferProvider(int name, AudioBufferProvider* bufferProvider 
{

    name -= TRACK0;

    ALOG_ASSERT(uint32_t(name) < MAX_NUM_TRACKS, "bad track name %d", name);



    if (mState.tracks[name].downmixerBufferProvider != NULL) {

        // update required?

        if (mState.tracks[name].downmixerBufferProvider->mTrackBufferProvider != bufferProvider) {

            ALOGV("AudioMixer::setBufferProvider(%p) for downmix", bufferProvider);

            // setting the buffer provider for a track that gets downmixed consists in:

            //  1/ setting the buffer provider to the "downmix / buffer provider" wrapper

            //     so it's the one that gets called when the buffer provider is needed,

            mState.tracks[name].bufferProvider = mState.tracks[name].downmixerBufferProvider;

            //  2/ saving the buffer provider for the track so the wrapper can use it

            //     when it downmixes.

            mState.tracks[name].downmixerBufferProvider->mTrackBufferProvider = bufferProvider;

        }

    } else {

        mState.tracks[name].bufferProvider = bufferProvider;

    }

}
@@ -419,13 +628,17 @@ void AudioMixer::process__validate(state_t* state, int64_t pts) 
                all16BitsStereoNoResample = false;

                resampling = true;

                t.hook = track__genericResample;

                ALOGV_IF((n & NEEDS_CHANNEL_COUNT__MASK) > NEEDS_CHANNEL_2,

                        "Track needs downmix + resample");

            } else {

                if ((n & NEEDS_CHANNEL_COUNT__MASK) == NEEDS_CHANNEL_1){

                    t.hook = track__16BitsMono;

                    all16BitsStereoNoResample = false;

                }

                if ((n & NEEDS_CHANNEL_COUNT__MASK) == NEEDS_CHANNEL_2){

                if ((n & NEEDS_CHANNEL_COUNT__MASK) >= NEEDS_CHANNEL_2){

                    t.hook = track__16BitsStereo;

                    ALOGV_IF((n & NEEDS_CHANNEL_COUNT__MASK) > NEEDS_CHANNEL_2,

                            "Track needs downmix");

                }

            }

        }