Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 4c3af933 authored by Shunkai Yao's avatar Shunkai Yao
Browse files

Enable HapticGenerator AIDL effect 

 - Add Haptic and Spatializer compatible check in AudioPolicyManager
 - The generated haptic samples is expected to append to the end of input
buffer which needs special handling in both effect and libaudiohal
 - Fix the haptic generator effect implementation wrong parameter and add
configuration with each init_params
 - Add parameter logging in HapticGenerator AIDL effect

Bug: 330686268
Test: atest --test-mapping hardware/interfaces/audio/aidl/vts:presubmit
Test: atest CtsMediaAudioTestCases
Test: HapticGenerator debug app on Pixel AIDL audio hal

Change-Id: Ia292acc54276d3ae035682ca421dc7ab7d6cff73
parent b5fbcfb0

media/libaudiohal/impl/EffectConversionHelperAidl.cpp

+14 −1
Original line number Diff line number Diff line
@@ -29,6 +29,7 @@ 
#include <system/audio_effects/effect_visualizer.h>



#include <utils/Log.h>

#include <Utils.h>



#include "EffectConversionHelperAidl.h"

#include "EffectProxy.h"
@@ -37,18 +38,20 @@ namespace android { 
namespace effect {



using ::aidl::android::aidl_utils::statusTFromBinderStatus;

using ::aidl::android::hardware::audio::common::getChannelCount;

using ::aidl::android::hardware::audio::effect::CommandId;

using ::aidl::android::hardware::audio::effect::Descriptor;

using ::aidl::android::hardware::audio::effect::Flags;

using ::aidl::android::hardware::audio::effect::IEffect;

using ::aidl::android::hardware::audio::effect::Parameter;

using ::aidl::android::hardware::audio::effect::State;

using ::aidl::android::media::audio::common::AudioChannelLayout;

using ::aidl::android::media::audio::common::AudioDeviceDescription;

using ::aidl::android::media::audio::common::AudioMode;

using ::aidl::android::media::audio::common::AudioSource;

using ::android::hardware::EventFlag;

using android::effect::utils::EffectParamReader;

using android::effect::utils::EffectParamWriter;

using android::hardware::EventFlag;



using ::android::status_t;
@@ -519,5 +522,15 @@ status_t EffectConversionHelperAidl::reopen() { 
    return OK;

}



size_t EffectConversionHelperAidl::getAudioChannelCount() const {

    return getChannelCount(mCommon.input.base.channelMask,

                           ~AudioChannelLayout::LAYOUT_HAPTIC_AB /* mask */);

}



size_t EffectConversionHelperAidl::getHapticChannelCount() const {

    return getChannelCount(mCommon.input.base.channelMask,

                           AudioChannelLayout::LAYOUT_HAPTIC_AB /* mask */);

}



}  // namespace effect

}  // namespace android

media/libaudiohal/impl/EffectConversionHelperAidl.h

+3 −0
Original line number Diff line number Diff line
@@ -49,6 +49,9 @@ class EffectConversionHelperAidl { 
    ::aidl::android::hardware::audio::effect::Descriptor getDescriptor() const;

    status_t reopen();



    size_t getAudioChannelCount() const;

    size_t getHapticChannelCount() const;



    uint8_t mOutputAccessMode = EFFECT_BUFFER_ACCESS_WRITE;



  protected:

media/libaudiohal/impl/EffectHalAidl.cpp

+32 −5
Original line number Diff line number Diff line
@@ -15,6 +15,7 @@ 
 */



#include <cstddef>

#include <cstring>

#define LOG_TAG "EffectHalAidl"

//#define LOG_NDEBUG 0
@@ -128,6 +129,7 @@ status_t EffectHalAidl::createAidlConversion( 
               ::aidl::android::hardware::audio::effect::getEffectTypeUuidHapticGenerator()) {

        mConversion = std::make_unique<android::effect::AidlConversionHapticGenerator>(

                effect, sessionId, ioId, desc, mIsProxyEffect);

        mIsHapticGenerator = true;

    } else if (typeUuid ==

               ::aidl::android::hardware::audio::effect::getEffectTypeUuidLoudnessEnhancer()) {

        mConversion = std::make_unique<android::effect::AidlConversionLoudnessEnhancer>(
@@ -200,7 +202,7 @@ status_t EffectHalAidl::process() { 
                              ::android::OK == efGroup->wait(kEventFlagDataMqUpdate, &efState,

                                                             1 /* ns */, true /* retry */) &&

                              efState & kEventFlagDataMqUpdate) {

        ALOGV("%s %s V%d receive dataMQUpdate eventFlag from HAL", __func__, effectName.c_str(),

        ALOGD("%s %s V%d receive dataMQUpdate eventFlag from HAL", __func__, effectName.c_str(),

              halVersion);



        mConversion->reopen();
@@ -216,7 +218,7 @@ status_t EffectHalAidl::process() { 
    }



    size_t available = inputQ->availableToWrite();

    size_t floatsToWrite = std::min(available, mInBuffer->getSize() / sizeof(float));

    const size_t floatsToWrite = std::min(available, mInBuffer->getSize() / sizeof(float));

    if (floatsToWrite == 0) {

        ALOGE("%s not able to write, floats in buffer %zu, space in FMQ %zu", __func__,

              mInBuffer->getSize() / sizeof(float), available);
@@ -248,7 +250,7 @@ status_t EffectHalAidl::process() { 
    }



    available = outputQ->availableToRead();

    size_t floatsToRead = std::min(available, mOutBuffer->getSize() / sizeof(float));

    const size_t floatsToRead = std::min(available, mOutBuffer->getSize() / sizeof(float));

    if (floatsToRead == 0) {

        ALOGE("%s not able to read, buffer space %zu, floats in FMQ %zu", __func__,

              mOutBuffer->getSize() / sizeof(float), available);
@@ -257,7 +259,8 @@ status_t EffectHalAidl::process() { 


    float *outputRawBuffer = mOutBuffer->audioBuffer()->f32;

    std::vector<float> tempBuffer;

    if (mConversion->mOutputAccessMode == EFFECT_BUFFER_ACCESS_ACCUMULATE) {

    // keep original data in the output buffer for accumulate mode or HapticGenerator effect

    if (mConversion->mOutputAccessMode == EFFECT_BUFFER_ACCESS_ACCUMULATE || mIsHapticGenerator) {

        tempBuffer.resize(floatsToRead);

        outputRawBuffer = tempBuffer.data();

    }
@@ -267,7 +270,31 @@ status_t EffectHalAidl::process() { 
              mOutBuffer->audioBuffer());

        return INVALID_OPERATION;

    }



    // HapticGenerator needs special handling because the generated haptic samples should append to

    // the end of audio samples, the generated haptic data pass back from HAL in output FMQ at same

    // offset as input buffer, here we skip the audio samples in output FMQ and append haptic

    // samples to the end of input buffer

    if (mIsHapticGenerator) {

        static constexpr float kHalFloatSampleLimit = 2.0f;

        assert(floatsToRead == floatsToWrite);

        const auto audioChNum = mConversion->getAudioChannelCount();

        const auto audioSamples =

                floatsToWrite * audioChNum / (audioChNum + mConversion->getHapticChannelCount());

        // accumulate or copy input to output, haptic samples remains all zero

        if (mConversion->mOutputAccessMode == EFFECT_BUFFER_ACCESS_ACCUMULATE) {

            accumulate_float(mOutBuffer->audioBuffer()->f32, mInBuffer->audioBuffer()->f32,

                             audioSamples);

        } else {

            memcpy_to_float_from_float_with_clamping(mOutBuffer->audioBuffer()->f32,

                                                     mInBuffer->audioBuffer()->f32, audioSamples,

                                                     kHalFloatSampleLimit);

        }

        // append the haptic sample at the end of input audio samples

        memcpy_to_float_from_float_with_clamping(mInBuffer->audioBuffer()->f32 + audioSamples,

                                                 outputRawBuffer + audioSamples,

                                                 floatsToRead - audioSamples, kHalFloatSampleLimit);

    } else if (mConversion->mOutputAccessMode == EFFECT_BUFFER_ACCESS_ACCUMULATE) {

        accumulate_float(mOutBuffer->audioBuffer()->f32, outputRawBuffer, floatsToRead);

    }

media/libaudiohal/impl/EffectHalAidl.h

+1 −0
Original line number Diff line number Diff line
@@ -73,6 +73,7 @@ class EffectHalAidl : public EffectHalInterface { 
    const int32_t mSessionId;

    const int32_t mIoId;

    const bool mIsProxyEffect;

    bool mIsHapticGenerator = false;



    std::unique_ptr<EffectConversionHelperAidl> mConversion;

media/libeffects/hapticgenerator/aidl/HapticGeneratorContext.cpp

+74 −33
Original line number Diff line number Diff line
@@ -21,23 +21,39 @@ 
#include <android-base/logging.h>

#include <android-base/parsedouble.h>

#include <android-base/properties.h>

#include <audio_utils/primitives.h>



#include "HapticGeneratorContext.h"



using aidl::android::hardware::audio::common::getChannelCount;

using aidl::android::hardware::audio::common::getPcmSampleSizeInBytes;

using aidl::android::media::audio::common::AudioChannelLayout;



namespace aidl::android::hardware::audio::effect {



HapticGeneratorContext::HapticGeneratorContext(int statusDepth, const Parameter::Common& common)

    : EffectContext(statusDepth, common) {

    mState = HAPTIC_GENERATOR_STATE_UNINITIALIZED;

    mSampleRate = common.input.base.sampleRate;

    mFrameCount = common.input.frameCount;

    init_params(common.input.base.channelMask, common.output.base.channelMask);



    mParams.mMaxVibratorScale = HapticGenerator::VibratorScale::MUTE;

    mParams.mVibratorInfo.resonantFrequencyHz = DEFAULT_RESONANT_FREQUENCY;

    mParams.mVibratorInfo.qFactor = DEFAULT_BSF_ZERO_Q;

    mParams.mVibratorInfo.maxAmplitude = 0.f;



    init_params(common);

    mState = HAPTIC_GENERATOR_STATE_INITIALIZED;

}



HapticGeneratorContext::~HapticGeneratorContext() {

    mState = HAPTIC_GENERATOR_STATE_UNINITIALIZED;

}



// Override EffectImpl::setCommon for HapticGenerator because we need init_params

RetCode HapticGeneratorContext::setCommon(const Parameter::Common& common) {

    init_params(common);

    return EffectContext::setCommon(common);

}



RetCode HapticGeneratorContext::enable() {

    if (mState != HAPTIC_GENERATOR_STATE_INITIALIZED) {

        return RetCode::ERROR_EFFECT_LIB_ERROR;
@@ -75,14 +91,15 @@ RetCode HapticGeneratorContext::setHgHapticScales( 
    for (const auto& [id, vibratorScale] : mParams.mHapticScales) {

        mParams.mMaxVibratorScale = std::max(mParams.mMaxVibratorScale, vibratorScale);

    }

    LOG(INFO) << " HapticGenerator VibratorScale set to " << toString(mParams.mMaxVibratorScale);

    return RetCode::SUCCESS;

}



HapticGenerator::VibratorInformation HapticGeneratorContext::getHgVibratorInformation() {

HapticGenerator::VibratorInformation HapticGeneratorContext::getHgVibratorInformation() const {

    return mParams.mVibratorInfo;

}



std::vector<HapticGenerator::HapticScale> HapticGeneratorContext::getHgHapticScales() {

std::vector<HapticGenerator::HapticScale> HapticGeneratorContext::getHgHapticScales() const {

    std::vector<HapticGenerator::HapticScale> result;

    for (const auto& [id, vibratorScale] : mParams.mHapticScales) {

        result.push_back({id, vibratorScale});
@@ -117,15 +134,8 @@ IEffect::Status HapticGeneratorContext::process(float* in, float* out, int sampl 
    auto frameSize = getInputFrameSize();

    RETURN_VALUE_IF(0 == frameSize, status, "zeroFrameSize");



    // The audio data must not be modified but just written to

    // output buffer according the access mode.

    if (in != out) {

        for (int i = 0; i < samples; i++) {

            out[i] = in[i];

        }

    }



    if (mState != HAPTIC_GENERATOR_STATE_ACTIVE) {

        LOG(WARNING) << " HapticGenerator in wrong state " << mState;

        return status;

    }
@@ -135,7 +145,8 @@ IEffect::Status HapticGeneratorContext::process(float* in, float* out, int sampl 
    }



    // Resize buffer if the haptic sample count is greater than buffer size.

    size_t hapticSampleCount = mFrameCount * mParams.mHapticChannelCount;

    const size_t hapticSampleCount = mFrameCount * mParams.mHapticChannelCount;

    const size_t audioSampleCount = mFrameCount * mParams.mAudioChannelCount;

    if (hapticSampleCount > mInputBuffer.size()) {

        // The inputBuffer and outputBuffer must have the same size, which must be at least

        // the haptic sample count.
@@ -155,45 +166,45 @@ IEffect::Status HapticGeneratorContext::process(float* in, float* out, int sampl 
            runProcessingChain(mInputBuffer.data(), mOutputBuffer.data(), mFrameCount);

    ::android::os::scaleHapticData(

            hapticOutBuffer, hapticSampleCount,

            {/*level=*/static_cast<::android::os::HapticLevel>(mParams.mMaxVibratorScale) },

            mParams.mVibratorInfo.qFactor);

            {static_cast<::android::os::HapticLevel>(mParams.mMaxVibratorScale)} /* scale */,

            mParams.mVibratorInfo.maxAmplitude /* limit */);



    // For haptic data, the haptic playback thread will copy the data from effect input

    // buffer, which contains haptic data at the end of the buffer, directly to sink buffer.

    // In that case, copy haptic data to input buffer instead of output buffer.

    // Note: this may not work with rpc/binder calls

    for (size_t i = 0; i < hapticSampleCount; ++i) {

        in[samples + i] = hapticOutBuffer[i];

    }

    return {STATUS_OK, samples, static_cast<int32_t>(samples + hapticSampleCount)};

    // In AIDL only output buffer is send back to the audio framework via FMQ. Here the effect copy

    // the generated haptic data to the target position of output buffer, the framework then append

    // it to the same position of input buffer.

    memcpy_to_float_from_float_with_clamping(out + audioSampleCount, hapticOutBuffer,

                                             hapticSampleCount, 2.f /* absMax */);

    return {STATUS_OK, samples, samples};

}



void HapticGeneratorContext::init_params(media::audio::common::AudioChannelLayout inputChMask,

                                         media::audio::common::AudioChannelLayout outputChMask) {

    mParams.mMaxVibratorScale = HapticGenerator::VibratorScale::MUTE;

    mParams.mVibratorInfo.resonantFrequencyHz = DEFAULT_RESONANT_FREQUENCY;

    mParams.mVibratorInfo.qFactor = DEFAULT_BSF_ZERO_Q;

void HapticGeneratorContext::init_params(const Parameter::Common& common) {

    mSampleRate = common.input.base.sampleRate;

    mFrameCount = common.input.frameCount;



    mParams.mAudioChannelCount = ::aidl::android::hardware::audio::common::getChannelCount(

            inputChMask, ~media::audio::common::AudioChannelLayout::LAYOUT_HAPTIC_AB);

            common.input.base.channelMask,

            ~media::audio::common::AudioChannelLayout::LAYOUT_HAPTIC_AB);

    mParams.mHapticChannelCount = ::aidl::android::hardware::audio::common::getChannelCount(

            outputChMask, media::audio::common::AudioChannelLayout::LAYOUT_HAPTIC_AB);

            common.output.base.channelMask,

            media::audio::common::AudioChannelLayout::LAYOUT_HAPTIC_AB);

    LOG_ALWAYS_FATAL_IF(mParams.mHapticChannelCount > 2, "haptic channel count is too large");

    for (int i = 0; i < mParams.mHapticChannelCount; ++i) {

        // By default, use the first audio channel to generate haptic channels.

        mParams.mHapticChannelSource[i] = 0;

    }



    mState = HAPTIC_GENERATOR_STATE_INITIALIZED;

    configure();

    LOG(DEBUG) << " HapticGenerator init context:\n" << contextToString();

}



float HapticGeneratorContext::getDistortionOutputGain() {

float HapticGeneratorContext::getDistortionOutputGain() const {

    float distortionOutputGain = getFloatProperty(

            "vendor.audio.hapticgenerator.distortion.output.gain", DEFAULT_DISTORTION_OUTPUT_GAIN);

    return distortionOutputGain;

}



float HapticGeneratorContext::getFloatProperty(const std::string& key, float defaultValue) {

float HapticGeneratorContext::getFloatProperty(const std::string& key, float defaultValue) const {

    float result;

    std::string value = ::android::base::GetProperty(key, "");

    if (!value.empty() && ::android::base::ParseFloat(value, &result)) {
@@ -322,4 +333,34 @@ float* HapticGeneratorContext::runProcessingChain(float* buf1, float* buf2, size 
    return in;

}



std::string HapticGeneratorContext::paramToString(const struct HapticGeneratorParam& param) const {

    std::stringstream ss;

    ss << "\t\ttHapticGenerator Parameters:\n";

    ss << "\t\t- mHapticChannelCount: " << param.mHapticChannelCount << '\n';

    ss << "\t\t- mAudioChannelCount: " << param.mAudioChannelCount << '\n';

    ss << "\t\t- mHapticChannelSource: " << param.mHapticChannelSource[0] << ", "

       << param.mHapticChannelSource[1] << '\n';

    ss << "\t\t- mMaxVibratorScale: " << ::android::internal::ToString(param.mMaxVibratorScale)

       << '\n';

    ss << "\t\t- mVibratorInfo: " << param.mVibratorInfo.toString() << '\n';

    for (const auto& it : param.mHapticScales)

        ss << "\t\t\t" << it.first << ": " << toString(it.second) << '\n';



    return ss.str();

}



std::string HapticGeneratorContext::contextToString() const {

    std::stringstream ss;

    ss << "\t\tHapticGenerator Context:\n";

    ss << "\t\t- state: " << mState << '\n';

    ss << "\t\t- bpf Q: " << DEFAULT_BPF_Q << '\n';

    ss << "\t\t- slow env normalization power: " << DEFAULT_SLOW_ENV_NORMALIZATION_POWER << '\n';

    ss << "\t\t- distortion corner frequency: " << DEFAULT_DISTORTION_CORNER_FREQUENCY << '\n';

    ss << "\t\t- distortion input gain: " << DEFAULT_DISTORTION_INPUT_GAIN << '\n';

    ss << "\t\t- distortion cube threshold: " << DEFAULT_DISTORTION_CUBE_THRESHOLD << '\n';

    ss << "\t\t- distortion output gain: " << getDistortionOutputGain() << '\n';

    ss << "\t\tHapticGenerator Parameters:\n" << paramToString(mParams) << "\n";

    return ss.str();

}



}  // namespace aidl::android::hardware::audio::effect