Merge "PreProcessing: Ensure only 10ms frames are processed at a time." into main (ca03a268) · Commits · e / os / android_frameworks_av

media/libeffects/preprocessing/aidl/PreProcessingContext.cpp

+37 −31

Original line number	Diff line number	Diff line
		@@ -16,6 +16,7 @@

		#include <cstddef>
		#define LOG_TAG "PreProcessingContext"
		#include <audio_utils/clock.h>
		#include <audio_utils/primitives.h>
		#include <Utils.h>

		@@ -26,6 +27,9 @@ namespace aidl::android::hardware::audio::effect {
		using aidl::android::media::audio::common::AudioDeviceDescription;
		using aidl::android::media::audio::common::AudioDeviceType;

		// Webrtc processes and returns 10ms data
		constexpr int WEBRTC_FRAME_LENGTH_MS = 10;

		RetCode PreProcessingContext::init(const Parameter::Common& common) {
		webrtc::AudioProcessingBuilder apBuilder;
		mAudioProcessingModule = apBuilder.Create();
		@@ -37,9 +41,7 @@ RetCode PreProcessingContext::init(const Parameter::Common& common) {
		updateConfigs(common);

		mEnabledMsk = 0;
		mProcessedMsk = 0;
		mRevEnabledMsk = 0;
		mRevProcessedMsk = 0;

		auto config = mAudioProcessingModule->GetConfig();
		switch (mType) {
		@@ -85,7 +87,6 @@ RetCode PreProcessingContext::enable() {
		config.echo_canceller.enabled = true;
		// AEC has reverse stream
		mRevEnabledMsk \|= typeMsk;
		mRevProcessedMsk = 0;
		break;
		case PreProcessingEffectType::AUTOMATIC_GAIN_CONTROL_V1:
		config.gain_controller1.enabled = true;
		@@ -97,7 +98,6 @@ RetCode PreProcessingContext::enable() {
		config.noise_suppression.enabled = true;
		break;
		}
		mProcessedMsk = 0;
		mAudioProcessingModule->ApplyConfig(config);
		mState = PRE_PROC_STATE_ACTIVE;
		return RetCode::SUCCESS;
		@@ -119,7 +119,6 @@ RetCode PreProcessingContext::disable() {
		config.echo_canceller.enabled = false;
		// AEC has reverse stream
		mRevEnabledMsk &= ~typeMsk;
		mRevProcessedMsk = 0;
		break;
		case PreProcessingEffectType::AUTOMATIC_GAIN_CONTROL_V1:
		config.gain_controller1.enabled = false;
		@@ -131,7 +130,6 @@ RetCode PreProcessingContext::disable() {
		config.noise_suppression.enabled = false;
		break;
		}
		mProcessedMsk = 0;
		mAudioProcessingModule->ApplyConfig(config);
		mState = PRE_PROC_STATE_INITIALIZED;
		return RetCode::SUCCESS;
		@@ -258,6 +256,12 @@ NoiseSuppression::Level PreProcessingContext::getNoiseSuppressionLevel() const {
		return mLevel;
		}

		int PreProcessingContext::calculateWebrtcChunkSizeInSamples() {
		return getCommon().input.base.sampleRate * WEBRTC_FRAME_LENGTH_MS / MILLIS_PER_SECOND *
		::aidl::android::hardware::audio::common::getChannelCount(
		mCommon.input.base.channelMask);
		}

		IEffect::Status PreProcessingContext::process(float* in, float* out, int samples) {
		IEffect::Status status = {EX_NULL_POINTER, 0, 0};
		RETURN_VALUE_IF(!in, status, "nullInput");
		@@ -268,18 +272,25 @@ IEffect::Status PreProcessingContext::process(float* in, float* out, int samples
		RETURN_VALUE_IF(inputFrameCount != outputFrameCount, status, "FrameCountMismatch");
		RETURN_VALUE_IF(0 == getInputFrameSize(), status, "zeroFrameSize");

		bool processEnable = (1 << int(mType) & mEnabledMsk);
		bool reverseProcessEnable = (((1 << int(mType)) & mRevEnabledMsk) &&
		(mType == PreProcessingEffectType::ACOUSTIC_ECHO_CANCELLATION));
		if (!(processEnable \|\| reverseProcessEnable)) {
		return {STATUS_OK, samples, samples};
		}
		const int processSamples = calculateWebrtcChunkSizeInSamples();
		std::vector<int16_t> in16(processSamples, 0);
		std::vector<int16_t> out16(processSamples, 0);

		// webrtc implementation clear out was_stream_delay_set every time after ProcessStream() call
		int samplesToProcess = std::min(samples, processSamples);
		for (int processedSamples = 0; processedSamples < samples;
		processedSamples += samplesToProcess) {
		samplesToProcess = std::min(samples - processedSamples, processSamples);
		// webrtc implementation clear out was_stream_delay_set every time after ProcessStream()
		// call
		mAudioProcessingModule->set_stream_delay_ms(mEchoDelayUs / 1000);

		std::vector<int16_t> in16(samples);
		std::vector<int16_t> out16(samples, 0);
		memcpy_to_i16_from_float(in16.data(), in, samples);

		mProcessedMsk \|= (1 << int(mType));

		if ((mProcessedMsk & mEnabledMsk) == mEnabledMsk) {
		mProcessedMsk = 0;
		memcpy_to_i16_from_float(in16.data(), in + processedSamples, samplesToProcess);
		if (processEnable) {
		int processStatus = mAudioProcessingModule->ProcessStream(in16.data(), mInputConfig,
		mOutputConfig, out16.data());
		if (processStatus != 0) {
		@@ -287,11 +298,7 @@ IEffect::Status PreProcessingContext::process(float* in, float* out, int samples
		return status;
		}
		}

		if (mType == PreProcessingEffectType::ACOUSTIC_ECHO_CANCELLATION) {
		mRevProcessedMsk \|= (1 << int(mType));
		if ((mRevProcessedMsk & mRevEnabledMsk) == mRevEnabledMsk) {
		mRevProcessedMsk = 0;
		if (reverseProcessEnable) {
		int revProcessStatus = mAudioProcessingModule->ProcessReverseStream(
		in16.data(), mInputConfig, mInputConfig, out16.data());
		if (revProcessStatus != 0) {
		@@ -299,10 +306,9 @@ IEffect::Status PreProcessingContext::process(float* in, float* out, int samples
		return status;
		}
		}
		memcpy_to_float_from_i16(out + processedSamples, out16.data(), samplesToProcess);
		}

		memcpy_to_float_from_i16(out, out16.data(), samples);

		return {STATUS_OK, samples, samples};
		}

media/libeffects/preprocessing/aidl/PreProcessingContext.h

+5 −7

Original line number	Diff line number	Diff line
		@@ -90,12 +90,8 @@ class PreProcessingContext final : public EffectContext {
		rtc::scoped_refptr<webrtc::AudioProcessing> mAudioProcessingModule;

		int mEnabledMsk; // bit field containing IDs of enabled pre processors
		int mProcessedMsk; // bit field containing IDs of pre processors already
		// processed in current round
		int mRevEnabledMsk; // bit field containing IDs of enabled pre processors
		// with reverse channel
		int mRevProcessedMsk; // bit field containing IDs of pre processors with
		// reverse channel already processed in current round

		webrtc::StreamConfig mInputConfig; // input stream configuration
		webrtc::StreamConfig mOutputConfig; // output stream configuration
		@@ -117,6 +113,8 @@ class PreProcessingContext final : public EffectContext {

		// NoiseSuppression
		NoiseSuppression::Level mLevel = NoiseSuppression::Level::LOW;

		int calculateWebrtcChunkSizeInSamples();
		};

		} // namespace aidl::android::hardware::audio::effect