Merge "Audio Eraser Effect placeholder" into main (9a2e74cb) · Commits · e / devices / smdk4412 / android_frameworks_av

media/libeffects/eraser/Android.bp

0 → 100644

+77 −0

Original line number	Diff line number	Diff line
		// Copyright (C) 2025 The Android Open Source Project
		//
		// Licensed under the Apache License, Version 2.0 (the "License");
		// you may not use this file except in compliance with the License.
		// You may obtain a copy of the License at
		//
		// http://www.apache.org/licenses/LICENSE-2.0
		//
		// Unless required by applicable law or agreed to in writing, software
		// distributed under the License is distributed on an "AS IS" BASIS,
		// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
		// See the License for the specific language governing permissions and
		// limitations under the License.

		package {
		default_applicable_licenses: [
		"frameworks_av_media_libeffects_eraser_license",
		],
		}

		license {
		name: "frameworks_av_media_libeffects_eraser_license",
		visibility: [":__subpackages__"],
		license_kinds: [
		"SPDX-license-identifier-Apache-2.0",
		],
		license_text: [
		"NOTICE",
		],
		}

		prebuilt_etc {
		name: "audio_eraser_classifier_model",
		src: "models/classifier.tflite",
		sub_dir: "models",
		filename: "classifier.tflite",
		vendor: true,
		}

		prebuilt_etc {
		name: "audio_eraser_separator_model",
		src: "models/separator.tflite",
		sub_dir: "models",
		filename: "separator.tflite",
		vendor: true,
		}

		cc_library_shared {
		name: "liberaser",
		srcs: [
		":effectCommonFile",
		"Eraser.cpp",
		"EraserContext.cpp",
		"LiteRTInstance.cpp",
		],
		defaults: [
		"aidlaudioeffectservice_defaults",
		],
		cflags: [
		"-Wall",
		"-Werror",
		"-Wextra",
		"-Wthread-safety",
		],
		header_libs: [
		"flatbuffer_headers",
		"libaudioeffects",
		"tensorflow_headers",
		],
		shared_libs: [
		"libtflite",
		],
		visibility: [
		"//hardware/interfaces/audio/aidl/default:__subpackages__",
		],
		relative_install_path: "soundfx",
		}

media/libeffects/eraser/Eraser.cpp

0 → 100644

+203 −0

Original line number	Diff line number	Diff line
		/*
		* Copyright (C) 2025 The Android Open Source Project
		*
		* Licensed under the Apache License, Version 2.0 (the "License");
		* you may not use this file except in compliance with the License.
		* You may obtain a copy of the License at
		*
		* http://www.apache.org/licenses/LICENSE-2.0
		*
		* Unless required by applicable law or agreed to in writing, software
		* distributed under the License is distributed on an "AS IS" BASIS,
		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
		* See the License for the specific language governing permissions and
		* limitations under the License.
		*/

		#define LOG_TAG "AHAL_EraserEffect"

		#include <android-base/logging.h>
		#include <aidl/android/hardware/audio/effect/Eraser.h>
		#include <system/audio_effects/effect_uuid.h>

		#include "Eraser.h"
		#include "EraserContext.h"

		#include <optional>

		using aidl::android::hardware::audio::common::getChannelCount;
		using aidl::android::hardware::audio::effect::Descriptor;
		using aidl::android::hardware::audio::effect::Eraser;
		using aidl::android::hardware::audio::effect::getEffectImplUuidEraser;
		using aidl::android::hardware::audio::effect::getEffectTypeUuidEraser;
		using aidl::android::hardware::audio::effect::IEffect;
		using aidl::android::hardware::audio::effect::State;
		using aidl::android::media::audio::common::AudioChannelLayout;
		using aidl::android::media::audio::common::AudioUuid;

		extern "C" binder_exception_t createEffect(const AudioUuid* in_impl_uuid,
		std::shared_ptr<IEffect>* instanceSpp) {
		if (!in_impl_uuid \|\| *in_impl_uuid != getEffectImplUuidEraser()) {
		LOG(ERROR) << __func__ << "uuid not supported";
		return EX_ILLEGAL_ARGUMENT;
		}

		if (!instanceSpp) {
		LOG(ERROR) << __func__ << " invalid input parameter!";
		return EX_ILLEGAL_ARGUMENT;
		}

		*instanceSpp = ndk::SharedRefBase::make<aidl::android::hardware::audio::effect::EraserImpl>();
		LOG(DEBUG) << __func__ << " instance " << instanceSpp->get() << " created";
		return EX_NONE;
		}

		extern "C" binder_exception_t queryEffect(const AudioUuid* in_impl_uuid, Descriptor* _aidl_return) {
		if (!in_impl_uuid \|\| *in_impl_uuid != getEffectImplUuidEraser()) {
		LOG(ERROR) << __func__ << "uuid not supported";
		return EX_ILLEGAL_ARGUMENT;
		}
		*_aidl_return = aidl::android::hardware::audio::effect::EraserImpl::kDescriptor;
		return EX_NONE;
		}

		namespace aidl::android::hardware::audio::effect {

		const std::string EraserImpl::kEffectName = "AOSP Audio Eraser";
		const Descriptor EraserImpl::kDescriptor = {
		.common = {.id = {.type = getEffectTypeUuidEraser(), .uuid = getEffectImplUuidEraser()},
		.flags = {.hwAcceleratorMode = Flags::HardwareAccelerator::NONE},
		.name = EraserImpl::kEffectName,
		.implementor = "The Android Open Source Project"}};

		ndk::ScopedAStatus EraserImpl::getDescriptor(Descriptor* _aidl_return) {
		LOG(DEBUG) << __func__ << kDescriptor.toString();
		*_aidl_return = kDescriptor;
		return ndk::ScopedAStatus::ok();
		}

		ndk::ScopedAStatus EraserImpl::setParameterSpecific(const Parameter::Specific& specific) {
		RETURN_IF(Parameter::Specific::eraser != specific.getTag(), EX_ILLEGAL_ARGUMENT,
		"EffectNotSupported");
		RETURN_IF(!mContext, EX_NULL_POINTER, "nullContext");

		auto param = specific.get<Parameter::Specific::eraser>();
		return mContext->setParam(param);
		}

		ndk::ScopedAStatus EraserImpl::getParameterSpecific(const Parameter::Id& id,
		Parameter::Specific* specific) {
		RETURN_IF(!mContext, EX_NULL_POINTER, "nullContext");

		auto tag = id.getTag();
		RETURN_IF(Parameter::Id::eraserTag != tag, EX_ILLEGAL_ARGUMENT, "wrongIdTag");
		auto eraserId = id.get<Parameter::Id::eraserTag>();
		auto eraserTag = eraserId.getTag();
		switch (eraserTag) {
		case Eraser::Id::commonTag: {
		auto specificTag = eraserId.get<Eraser::Id::commonTag>();
		std::optional<Eraser> param = mContext->getParam(specificTag);
		if (!param.has_value()) {
		return ndk::ScopedAStatus::fromExceptionCodeWithMessage(EX_ILLEGAL_ARGUMENT,
		"EraserTagNotSupported");
		}
		specific->set<Parameter::Specific::eraser>(param.value());
		break;
		}
		default: {
		LOG(ERROR) << __func__ << " unsupported tag: " << toString(tag);
		return ndk::ScopedAStatus::fromExceptionCodeWithMessage(EX_ILLEGAL_ARGUMENT,
		"EraserTagNotSupported");
		}
		}
		return ndk::ScopedAStatus::ok();
		}

		std::shared_ptr<EffectContext> EraserImpl::createContext(const Parameter::Common& common) {
		if (mContext) {
		LOG(DEBUG) << __func__ << " context already exist";
		} else {
		mContext = std::make_shared<EraserContext>(1 /* statusFmqDepth */, common);
		}
		return mContext;
		}

		RetCode EraserImpl::releaseContext() {
		if (mContext) {
		mContext.reset();
		}
		return RetCode::SUCCESS;
		}

		EraserImpl::~EraserImpl() {
		cleanUp();
		LOG(DEBUG) << __func__;
		}

		ndk::ScopedAStatus EraserImpl::command(CommandId command) {
		std::lock_guard lg(mImplMutex);
		RETURN_IF(mState == State::INIT, EX_ILLEGAL_STATE, "instanceNotOpen");

		switch (command) {
		case CommandId::START:
		RETURN_OK_IF(mState == State::PROCESSING);
		mState = State::PROCESSING;
		mContext->enable();
		startThread();
		RETURN_IF(notifyEventFlag(mDataMqNotEmptyEf) != RetCode::SUCCESS, EX_ILLEGAL_STATE,
		"notifyEventFlagNotEmptyFailed");
		break;
		case CommandId::STOP:
		RETURN_OK_IF(mState == State::IDLE \|\| mState == State::DRAINING);
		if (mVersion < kDrainSupportedVersion) {
		mState = State::IDLE;
		stopThread();
		mContext->disable();
		} else {
		mState = State::DRAINING;
		startDraining();
		mContext->startDraining();
		}
		RETURN_IF(notifyEventFlag(mDataMqNotEmptyEf) != RetCode::SUCCESS, EX_ILLEGAL_STATE,
		"notifyEventFlagNotEmptyFailed");
		break;
		case CommandId::RESET:
		mState = State::IDLE;
		RETURN_IF(notifyEventFlag(mDataMqNotEmptyEf) != RetCode::SUCCESS, EX_ILLEGAL_STATE,
		"notifyEventFlagNotEmptyFailed");
		stopThread();
		mImplContext->disable();
		mImplContext->reset();
		mImplContext->resetBuffer();
		break;
		default:
		LOG(ERROR) << getEffectNameWithVersion() << __func__ << " instance still processing";
		return ndk::ScopedAStatus::fromExceptionCodeWithMessage(EX_ILLEGAL_ARGUMENT,
		"CommandIdNotSupported");
		}
		LOG(VERBOSE) << getEffectNameWithVersion() << __func__
		<< " transfer to state: " << toString(mState);
		return ndk::ScopedAStatus::ok();
		}

		// Processing method running in EffectWorker thread.
		IEffect::Status EraserImpl::effectProcessImpl(float* in, float* out, int samples) {
		RETURN_VALUE_IF(!mContext, (IEffect::Status{EX_NULL_POINTER, 0, 0}), "nullContext");
		IEffect::Status procStatus{STATUS_NOT_ENOUGH_DATA, 0, 0};
		procStatus = mContext->process(in, out, samples);
		if (mState == State::DRAINING && procStatus.status == STATUS_NOT_ENOUGH_DATA) {
		drainingComplete_l();
		}

		return procStatus;
		}

		void EraserImpl::drainingComplete_l() {
		if (mState != State::DRAINING) return;

		LOG(DEBUG) << getEffectNameWithVersion() << __func__;
		finishDraining();
		mState = State::IDLE;
		}

		} // namespace aidl::android::hardware::audio::effect

media/libeffects/eraser/Eraser.h

0 → 100644

+60 −0

Original line number	Diff line number	Diff line
		/*
		* Copyright (C) 2025 The Android Open Source Project
		*
		* Licensed under the Apache License, Version 2.0 (the "License");
		* you may not use this file except in compliance with the License.
		* You may obtain a copy of the License at
		*
		* http://www.apache.org/licenses/LICENSE-2.0
		*
		* Unless required by applicable law or agreed to in writing, software
		* distributed under the License is distributed on an "AS IS" BASIS,
		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
		* See the License for the specific language governing permissions and
		* limitations under the License.
		*/

		#pragma once

		#include <string>
		#include <vector>

		#include <fmq/AidlMessageQueue.h>

		#include "effect-impl/EffectContext.h"
		#include "effect-impl/EffectImpl.h"

		#include "EraserContext.h"

		namespace aidl::android::hardware::audio::effect {

		class EraserImpl final : public EffectImpl {
		public:
		~EraserImpl() final;

		static const std::string kEffectName;
		static const Capability kCapability;
		static const Descriptor kDescriptor;

		ndk::ScopedAStatus getDescriptor(Descriptor* _aidl_return) final;
		ndk::ScopedAStatus setParameterSpecific(const Parameter::Specific& specific)
		REQUIRES(mImplMutex) final;
		ndk::ScopedAStatus getParameterSpecific(const Parameter::Id& id, Parameter::Specific* specific)
		REQUIRES(mImplMutex) final;

		std::shared_ptr<EffectContext> createContext(const Parameter::Common& common)
		REQUIRES(mImplMutex) final;
		RetCode releaseContext() REQUIRES(mImplMutex) final;

		std::string getEffectName() final { return kEffectName; };
		IEffect::Status effectProcessImpl(float* in, float* out, int samples)
		REQUIRES(mImplMutex) final;

		ndk::ScopedAStatus command(CommandId command) final EXCLUDES(mImplMutex);
		void drainingComplete_l() REQUIRES(mImplMutex);

		private:
		static const std::vector<Range::SpatializerRange> kRanges;
		std::shared_ptr<EraserContext> mContext GUARDED_BY(mImplMutex);
		};
		} // namespace aidl::android::hardware::audio::effect

media/libeffects/eraser/EraserContext.cpp

0 → 100644

+138 −0

Original line number	Diff line number	Diff line
		/*
		* Copyright (C) 2025 The Android Open Source Project
		*
		* Licensed under the Apache License, Version 2.0 (the "License");
		* you may not use this file except in compliance with the License.
		* You may obtain a copy of the License at
		*
		* http://www.apache.org/licenses/LICENSE-2.0
		*
		* Unless required by applicable law or agreed to in writing, software
		* distributed under the License is distributed on an "AS IS" BASIS,
		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
		* See the License for the specific language governing permissions and
		* limitations under the License.
		*/

		#define LOG_TAG "AHAL_EraserContext"

		#include <android-base/logging.h>
		#include <sys/param.h>
		#include <sys/stat.h>

		#include "EraserContext.h"
		#include "LiteRTInstance.h"

		using aidl::android::media::audio::eraser::Mode;

		namespace aidl::android::hardware::audio::effect {

		const EraserContext::EraserConfiguration EraserContext::kDefaultConfig = {
		.mode = Mode::ERASER};
		const EraserContext::EraserCapability EraserContext::kCapability = {
		.modes = {Mode::ERASER, Mode::CLASSIFIER}};

		EraserContext::EraserContext(int statusDepth, const Parameter::Common& common)
		: EffectContext(statusDepth, common),
		mCommon(common),
		mConfig(kDefaultConfig) {
		LOG(DEBUG) << __func__ << ": Creating EraserContext";
		init();
		}

		EraserContext::~EraserContext() {
		LOG(DEBUG) << __func__ << ": Destroying EraserContext";
		}

		void EraserContext::init() {
		mChannelCount = static_cast<int>(::aidl::android::hardware::audio::common::getChannelCount(
		mCommon.input.base.channelMask));
		}

		RetCode EraserContext::enable() {
		if (mConfig.mode == Mode::ERASER) {
		if (!mSeparatorInstance) {
		mSeparatorInstance = std::make_unique<LiteRTInstance>(kSeparatorModelPath);
		}
		if (mSeparatorInstance && mSeparatorInstance->initialize()) {
		mSeparatorInstance->warmup();
		} else {
		LOG(ERROR) << __func__ << ": failed to enable separator";
		return RetCode::ERROR_EFFECT_LIB_ERROR;
		}
		} else {
		mSeparatorInstance.reset();
		}

		if (!mClassifierInstance) {
		mClassifierInstance = std::make_unique<LiteRTInstance>(kClassifierModelPath);
		}
		if (mClassifierInstance && mClassifierInstance->initialize()) {
		mClassifierInstance->warmup();
		} else {
		LOG(ERROR) << __func__ << ": failed to enable classifier";
		return RetCode::ERROR_EFFECT_LIB_ERROR;
		}

		return RetCode::SUCCESS;
		}

		RetCode EraserContext::disable() {
		mClassifierInstance.reset();
		mSeparatorInstance.reset();
		return RetCode::SUCCESS;
		}

		RetCode EraserContext::reset() {
		// reset model inference indexes
		if (mSeparatorInstance) mSeparatorInstance->resetTensorIndex();
		if (mClassifierInstance) mClassifierInstance->resetTensorIndex();
		return RetCode::SUCCESS;
		}

		std::optional<Eraser> EraserContext::getParam(Eraser::Tag tag) {
		switch (tag) {
		case Eraser::capability:
		return Eraser::make<Eraser::capability>(kCapability);
		case Eraser::configuration:
		return Eraser::make<Eraser::configuration>(mConfig);
		default: {
		LOG(ERROR) << __func__ << " unsupported tag: " << toString(tag);
		return std::nullopt;
		}
		}
		}

		ndk::ScopedAStatus EraserContext::setParam(Eraser eraser) {
		const auto tag = eraser.getTag();
		switch (tag) {
		case Eraser::configuration: {
		mConfig = eraser.get<Eraser::configuration>();
		return ndk::ScopedAStatus::ok();
		}
		default: {
		LOG(ERROR) << __func__ << " unsupported tag: " << toString(tag);
		return ndk::ScopedAStatus::fromExceptionCode(EX_ILLEGAL_ARGUMENT);
		}
		}
		}

		IEffect::Status EraserContext::process(float, float, int samples) {
		IEffect::Status procStatus = {EX_ILLEGAL_ARGUMENT, 0, 0};
		const auto inputChCount = common::getChannelCount(mCommon.input.base.channelMask);
		const auto outputChCount = common::getChannelCount(mCommon.output.base.channelMask);
		if (inputChCount < outputChCount) {
		LOG(ERROR) << __func__ << " invalid channel count, in: " << inputChCount
		<< " out: " << outputChCount;
		return procStatus;
		}

		// TODO: convert input buffer to tensor input format (16kHz/mono/float16) and process

		const int iFrames = samples / inputChCount;
		procStatus.fmqConsumed = static_cast<int32_t>(iFrames * inputChCount);
		procStatus.fmqProduced = static_cast<int32_t>(iFrames * outputChCount);
		return procStatus;
		}

		} // namespace aidl::android::hardware::audio::effect

media/libeffects/eraser/EraserContext.h

0 → 100644

+72 −0

Original line number	Diff line number	Diff line
		/*
		* Copyright (C) 2025 The Android Open Source Project
		*
		* Licensed under the Apache License, Version 2.0 (the "License");
		* you may not use this file except in compliance with the License.
		* You may obtain a copy of the License at
		*
		* http://www.apache.org/licenses/LICENSE-2.0
		*
		* Unless required by applicable law or agreed to in writing, software
		* distributed under the License is distributed on an "AS IS" BASIS,
		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
		* See the License for the specific language governing permissions and
		* limitations under the License.
		*/

		#pragma once

		#include <memory>
		#include <optional>
		#include <string>

		#include <aidl/android/hardware/audio/effect/Eraser.h>
		#include <aidl/android/media/audio/eraser/Capability.h>
		#include <aidl/android/media/audio/eraser/Configuration.h>

		#include "effect-impl/EffectContext.h"
		#include "LiteRTInstance.h"

		namespace aidl::android::hardware::audio::effect {

		class EraserContext final : public EffectContext {
		public:
		EraserContext(int statusDepth, const Parameter::Common& common);
		~EraserContext() final;

		RetCode enable() override;
		RetCode disable() override;
		RetCode reset() override;

		std::optional<Eraser> getParam(Eraser::Tag tag);
		ndk::ScopedAStatus setParam(Eraser eraser);
		IEffect::Status process(float* in, float* out, int samples);
		using EraserConfiguration = android::media::audio::eraser::Configuration;
		using EraserCapability = android::media::audio::eraser::Capability;

		private:
		static const EraserConfiguration kDefaultConfig;

		static const EraserCapability kCapability;

		// yamnet model was used for classifier:
		// https://github.com/tensorflow/models/tree/master/research/audioset/yamnet
		const std::string kClassifierModelPath =
		"/apex/com.android.hardware.audio/etc/models/classifier.tflite";

		// neurips2020_mixit model was used for separator:
		// https://github.com/google-research/sound-separation/tree/master/models/neurips2020_mixit
		const std::string kSeparatorModelPath =
		"/apex/com.android.hardware.audio/etc/models/separator.tflite";

		int mChannelCount;
		Parameter::Common mCommon;
		EraserConfiguration mConfig;

		std::unique_ptr<LiteRTInstance> mClassifierInstance;
		std::unique_ptr<LiteRTInstance> mSeparatorInstance;

		void init();
		};

		} // namespace aidl::android::hardware::audio::effect
		No newline at end of file