Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 245cdd91 authored by jiabin's avatar jiabin
Browse files

Support audio-haptic coupled playback. 

When trying to play with haptic channel mask, use adjust channels
buffer provider to make the haptic channel the same as the output one.
If haptic playback is supported, use adjust channel non destructive
buffer provider to output haptic data to the end of the sink buffer.
Otherwise, haptic data will be ignored.

Test: Manually
Bug: 111454766
Change-Id: Ic5f780de48c1e71de6ba5c4774d1ed2e9c8c51a0
parent dce8f8cc

media/libaudioclient/include/media/AudioMixer.h

+8 −0
Original line number Diff line number Diff line
@@ -78,6 +78,8 @@ public: 
        DOWNMIX_TYPE    = 0X4004,

        MIXER_FORMAT    = 0x4005, // AUDIO_FORMAT_PCM_(FLOAT|16_BIT)

        MIXER_CHANNEL_MASK = 0x4006, // Channel mask for mixer output

        // for haptic

        HAPTIC_ENABLED  = 0x4007, // Set haptic data from this track should be played or not.

        // for target RESAMPLE

        SAMPLE_RATE     = 0x4100, // Configure sample rate conversion on this track name;

                                  // parameter 'value' is the new sample rate in Hz.
@@ -329,6 +331,7 @@ private: 
         * 7) mTimestretchBufferProvider: Adds timestretching for playback rate

         */

        AudioBufferProvider*     mInputBufferProvider;    // externally provided buffer provider.

        // TODO: combine AdjustChannelsBufferProvider and AdjustChannelsNonDestructiveBufferProvider

        std::unique_ptr<PassthruBufferProvider> mAdjustChannelsBufferProvider;

        std::unique_ptr<PassthruBufferProvider> mAdjustChannelsNonDestructiveBufferProvider;

        std::unique_ptr<PassthruBufferProvider> mReformatBufferProvider;
@@ -360,6 +363,11 @@ private: 
        AudioPlaybackRate    mPlaybackRate;



        // Haptic

        bool                 mHapticPlaybackEnabled;

        audio_channel_mask_t mHapticChannelMask;

        uint32_t             mHapticChannelCount;

        audio_channel_mask_t mMixerHapticChannelMask;

        uint32_t             mMixerHapticChannelCount;

        uint32_t             mAdjustInChannelCount;

        uint32_t             mAdjustOutChannelCount;

        uint32_t             mAdjustNonDestructiveInChannelCount;

media/libaudioprocessing/AudioMixer.cpp

+52 −10
Original line number Diff line number Diff line
@@ -136,6 +136,9 @@ status_t AudioMixer::create( 


        // no initialization needed

        // t->frameCount

        t->mHapticChannelMask = channelMask & AUDIO_CHANNEL_HAPTIC_ALL;

        t->mHapticChannelCount = audio_channel_count_from_out_mask(t->mHapticChannelMask);

        channelMask &= ~AUDIO_CHANNEL_HAPTIC_ALL;

        t->channelCount = audio_channel_count_from_out_mask(channelMask);

        t->enabled = false;

        ALOGV_IF(audio_channel_mask_get_bits(channelMask) != AUDIO_CHANNEL_OUT_STEREO,
@@ -163,10 +166,13 @@ status_t AudioMixer::create( 
        t->mMixerChannelCount = audio_channel_count_from_out_mask(t->mMixerChannelMask);

        t->mPlaybackRate = AUDIO_PLAYBACK_RATE_DEFAULT;

        // haptic

        t->mAdjustInChannelCount = 0;

        t->mAdjustOutChannelCount = 0;

        t->mAdjustNonDestructiveInChannelCount = 0;

        t->mAdjustNonDestructiveOutChannelCount = 0;

        t->mHapticPlaybackEnabled = false;

        t->mMixerHapticChannelMask = AUDIO_CHANNEL_NONE;

        t->mMixerHapticChannelCount = 0;

        t->mAdjustInChannelCount = t->channelCount + t->mHapticChannelCount;

        t->mAdjustOutChannelCount = t->channelCount + t->mMixerHapticChannelCount;

        t->mAdjustNonDestructiveInChannelCount = t->mAdjustOutChannelCount;

        t->mAdjustNonDestructiveOutChannelCount = t->channelCount;

        t->mKeepContractedChannels = false;

        // Check the downmixing (or upmixing) requirements.

        status_t status = t->prepareForDownmix();
@@ -193,13 +199,20 @@ bool AudioMixer::setChannelMasks(int name, 
    LOG_ALWAYS_FATAL_IF(!exists(name), "invalid name: %d", name);

    const std::shared_ptr<Track> &track = mTracks[name];



    if (trackChannelMask == track->channelMask

            && mixerChannelMask == track->mMixerChannelMask) {

    if (trackChannelMask == (track->channelMask | track->mHapticChannelMask)

            && mixerChannelMask == (track->mMixerChannelMask | track->mMixerHapticChannelMask)) {

        return false;  // no need to change

    }

    const audio_channel_mask_t hapticChannelMask = trackChannelMask & AUDIO_CHANNEL_HAPTIC_ALL;

    trackChannelMask &= ~AUDIO_CHANNEL_HAPTIC_ALL;

    const audio_channel_mask_t mixerHapticChannelMask = mixerChannelMask & AUDIO_CHANNEL_HAPTIC_ALL;

    mixerChannelMask &= ~AUDIO_CHANNEL_HAPTIC_ALL;

    // always recompute for both channel masks even if only one has changed.

    const uint32_t trackChannelCount = audio_channel_count_from_out_mask(trackChannelMask);

    const uint32_t mixerChannelCount = audio_channel_count_from_out_mask(mixerChannelMask);

    const uint32_t hapticChannelCount = audio_channel_count_from_out_mask(hapticChannelMask);

    const uint32_t mixerHapticChannelCount =

            audio_channel_count_from_out_mask(mixerHapticChannelMask);



    ALOG_ASSERT((trackChannelCount <= MAX_NUM_CHANNELS_TO_DOWNMIX)

            && trackChannelCount
@@ -208,6 +221,24 @@ bool AudioMixer::setChannelMasks(int name, 
    track->channelCount = trackChannelCount;

    track->mMixerChannelMask = mixerChannelMask;

    track->mMixerChannelCount = mixerChannelCount;

    track->mHapticChannelMask = hapticChannelMask;

    track->mHapticChannelCount = hapticChannelCount;

    track->mMixerHapticChannelMask = mixerHapticChannelMask;

    track->mMixerHapticChannelCount = mixerHapticChannelCount;



    if (track->mHapticChannelCount > 0) {

        track->mAdjustInChannelCount = track->channelCount + track->mHapticChannelCount;

        track->mAdjustOutChannelCount = track->channelCount + track->mMixerHapticChannelCount;

        track->mAdjustNonDestructiveInChannelCount = track->mAdjustOutChannelCount;

        track->mAdjustNonDestructiveOutChannelCount = track->channelCount;

        track->mKeepContractedChannels = track->mHapticPlaybackEnabled;

    } else {

        track->mAdjustInChannelCount = 0;

        track->mAdjustOutChannelCount = 0;

        track->mAdjustNonDestructiveInChannelCount = 0;

        track->mAdjustNonDestructiveOutChannelCount = 0;

        track->mKeepContractedChannels = false;

    }



    // channel masks have changed, does this track need a downmixer?

    // update to try using our desired format (if we aren't already using it)
@@ -616,7 +647,8 @@ void AudioMixer::setParameter(int name, int target, int param, void *value) 
        case CHANNEL_MASK: {

            const audio_channel_mask_t trackChannelMask =

                static_cast<audio_channel_mask_t>(valueInt);

            if (setChannelMasks(name, trackChannelMask, track->mMixerChannelMask)) {

            if (setChannelMasks(name, trackChannelMask,

                    (track->mMixerChannelMask | track->mMixerHapticChannelMask))) {

                ALOGV("setParameter(TRACK, CHANNEL_MASK, %x)", trackChannelMask);

                invalidate();

            }
@@ -665,11 +697,21 @@ void AudioMixer::setParameter(int name, int target, int param, void *value) 
        case MIXER_CHANNEL_MASK: {

            const audio_channel_mask_t mixerChannelMask =

                    static_cast<audio_channel_mask_t>(valueInt);

            if (setChannelMasks(name, track->channelMask, mixerChannelMask)) {

            if (setChannelMasks(name, track->channelMask | track->mHapticChannelMask,

                    mixerChannelMask)) {

                ALOGV("setParameter(TRACK, MIXER_CHANNEL_MASK, %#x)", mixerChannelMask);

                invalidate();

            }

            } break;

        case HAPTIC_ENABLED: {

            const bool hapticPlaybackEnabled = static_cast<bool>(valueInt);

            if (track->mHapticPlaybackEnabled != hapticPlaybackEnabled) {

                track->mHapticPlaybackEnabled = hapticPlaybackEnabled;

                track->mKeepContractedChannels = hapticPlaybackEnabled;

                track->prepareForAdjustChannelsNonDestructive(mFrameCount);

                track->prepareForAdjustChannels();

            }

            } break;

        default:

            LOG_ALWAYS_FATAL("setParameter track: bad param %d", param);

        }
@@ -1359,8 +1401,8 @@ void AudioMixer::process__nop() 


        const std::shared_ptr<Track> &t = mTracks[group[0]];

        memset(t->mainBuffer, 0,

                mFrameCount * t->mMixerChannelCount

                * audio_bytes_per_sample(t->mMixerFormat));

                mFrameCount * audio_bytes_per_frame(

                        t->mMixerChannelCount + t->mMixerHapticChannelCount, t->mMixerFormat));



        // now consume data

        for (const int name : group) {

services/audioflinger/FastMixer.cpp

+20 −4
Original line number Diff line number Diff line
@@ -37,8 +37,9 @@ 
#include <cpustats/ThreadCpuUsage.h>

#endif

#endif

#include <audio_utils/mono_blend.h>

#include <audio_utils/channels.h>

#include <audio_utils/format.h>

#include <audio_utils/mono_blend.h>

#include <media/AudioMixer.h>

#include "FastMixer.h"

#include "TypedLogger.h"
@@ -159,21 +160,25 @@ void FastMixer::onStateChange() 
    if (current->mOutputSinkGen != mOutputSinkGen) {

        mOutputSink = current->mOutputSink;

        mOutputSinkGen = current->mOutputSinkGen;

        mSinkChannelMask = current->mSinkChannelMask;

        if (mOutputSink == NULL) {

            mFormat = Format_Invalid;

            mSampleRate = 0;

            mSinkChannelCount = 0;

            mSinkChannelMask = AUDIO_CHANNEL_NONE;

            mAudioChannelCount = 0;

        } else {

            mFormat = mOutputSink->format();

            mSampleRate = Format_sampleRate(mFormat);

            mSinkChannelCount = Format_channelCount(mFormat);

            LOG_ALWAYS_FATAL_IF(mSinkChannelCount > AudioMixer::MAX_NUM_CHANNELS);



            // TODO: Add channel mask to NBAIO_Format

            // We assume that the channel mask must be a valid positional channel mask.

            if (mSinkChannelMask == AUDIO_CHANNEL_NONE) {

                mSinkChannelMask = audio_channel_out_mask_from_count(mSinkChannelCount);

            }

            mAudioChannelCount = mSinkChannelCount - audio_channel_count_from_out_mask(

                    mSinkChannelMask & AUDIO_CHANNEL_HAPTIC_ALL);

        }

        dumpState->mSampleRate = mSampleRate;

    }
@@ -288,6 +293,8 @@ void FastMixer::onStateChange() 
                        (void *)(uintptr_t)fastTrack->mChannelMask);

                mMixer->setParameter(name, AudioMixer::TRACK, AudioMixer::MIXER_CHANNEL_MASK,

                        (void *)(uintptr_t)mSinkChannelMask);

                mMixer->setParameter(name, AudioMixer::TRACK, AudioMixer::HAPTIC_ENABLED,

                        (void *)(uintptr_t)fastTrack->mHapticPlaybackEnabled);

                mMixer->enable(name);

            }

            mGenerations[i] = fastTrack->mGeneration;
@@ -324,6 +331,8 @@ void FastMixer::onStateChange() 
                            (void *)(uintptr_t)fastTrack->mChannelMask);

                    mMixer->setParameter(name, AudioMixer::TRACK, AudioMixer::MIXER_CHANNEL_MASK,

                            (void *)(uintptr_t)mSinkChannelMask);

                    mMixer->setParameter(name, AudioMixer::TRACK, AudioMixer::HAPTIC_ENABLED,

                            (void *)(uintptr_t)fastTrack->mHapticPlaybackEnabled);

                    // already enabled

                }

                mGenerations[i] = fastTrack->mGeneration;
@@ -468,6 +477,13 @@ void FastMixer::onWork() 
            memcpy_by_audio_format(buffer, mFormat.mFormat, mMixerBuffer, mMixerBufferFormat,

                    frameCount * Format_channelCount(mFormat));

        }

        if (mSinkChannelMask & AUDIO_CHANNEL_HAPTIC_ALL) {

            // When there are haptic channels, the sample data is partially interleaved.

            // Make the sample data fully interleaved here.

            adjust_channels_non_destructive(buffer, mAudioChannelCount, buffer, mSinkChannelCount,

                    audio_bytes_per_sample(mFormat.mFormat),

                    frameCount * audio_bytes_per_frame(mAudioChannelCount, mFormat.mFormat));

        }

        // if non-NULL, then duplicate write() to this non-blocking sink

#ifdef TEE_SINK

        mTee.write(buffer, frameCount);

services/audioflinger/FastMixer.h

+2 −0
Original line number Diff line number Diff line
@@ -76,6 +76,8 @@ private: 
    size_t          mMixerBufferSize;

    audio_format_t  mMixerBufferFormat; // mixer output format: AUDIO_FORMAT_PCM_(16_BIT|FLOAT).



    uint32_t        mAudioChannelCount; // audio channel count, excludes haptic channels.



    enum {UNDEFINED, MIXED, ZEROED} mMixerBufferState;

    NBAIO_Format    mFormat;

    unsigned        mSampleRate;

services/audioflinger/FastMixerState.h

+4 −0
Original line number Diff line number Diff line
@@ -47,6 +47,7 @@ struct FastTrack { 
    audio_channel_mask_t    mChannelMask;    // AUDIO_CHANNEL_OUT_MONO or AUDIO_CHANNEL_OUT_STEREO

    audio_format_t          mFormat;         // track format

    int                     mGeneration;     // increment when any field is assigned

    bool                    mHapticPlaybackEnabled = false; // haptic playback is enabled or not

};



// Represents a single state of the fast mixer
@@ -69,6 +70,9 @@ struct FastMixerState : FastThreadState { 
    NBAIO_Sink* mOutputSink;    // HAL output device, must already be negotiated

    int         mOutputSinkGen; // increment when mOutputSink is assigned

    size_t      mFrameCount;    // number of frames per fast mix buffer

    audio_channel_mask_t mSinkChannelMask; // If not AUDIO_CHANNEL_NONE, specifies sink channel

                                           // mask when it cannot be directly calculated from

                                           // channel count



    // Extends FastThreadState::Command

    static const Command