AudioFlinger: compute MEL values on framework level (b042ee60) · Commits · e / os / android_frameworks_av

services/audioflinger/Android.bp

+2 −0

Original line number	Diff line number	Diff line
		@@ -43,6 +43,7 @@ cc_library_shared {
		"FastThread.cpp",
		"FastThreadDumpState.cpp",
		"FastThreadState.cpp",
		"MelReporter.cpp",
		"NBAIO_Tee.cpp",
		"PatchCommandThread.cpp",
		"PatchPanel.cpp",
		@@ -100,6 +101,7 @@ cc_library_shared {
		"libaaudio_headers",
		"libaudioclient_headers",
		"libaudiohal_headers",
		"libaudioutils_headers",
		"libmedia_headers",
		],

services/audioflinger/AudioFlinger.cpp

+4 −0

Original line number	Diff line number	Diff line
		@@ -325,6 +325,7 @@ AudioFlinger::AudioFlinger()
		mPatchPanel(this),
		mPatchCommandThread(sp<PatchCommandThread>::make()),
		mDeviceEffectManager(sp<DeviceEffectManager>::make(*this)),
		mMelReporter(sp<MelReporter>::make(*this)),
		mSystemReady(false)
		{
		// Move the audio session unique ID generator start base as time passes to limit risk of
		@@ -876,6 +877,9 @@ status_t AudioFlinger::dump(int fd, const Vector<String16>& args)

		mDeviceEffectManager->dump(fd);

		std::string melOutput = mMelReporter->dump();
		write(fd, melOutput.c_str(), melOutput.size());

		// dump external setParameters
		auto dumpLogger = [fd](SimpleLog& logger, const char* name) {
		dprintf(fd, "\n%s setParameters:\n", name);

services/audioflinger/AudioFlinger.h

+7 −2

Original line number	Diff line number	Diff line
		@@ -82,6 +82,8 @@
		#include <audio_utils/clock.h>
		#include <audio_utils/FdToString.h>
		#include <audio_utils/LinearMap.h>
		#include <audio_utils/MelAggregator.h>
		#include <audio_utils/MelProcessor.h>
		#include <audio_utils/SimpleLog.h>
		#include <audio_utils/TimestampVerifier.h>

		@@ -632,12 +634,14 @@ using effect_buffer_t = int16_t;

		#include "PatchPanel.h"

		#include "Effects.h"

		#include "PatchCommandThread.h"

		#include "Effects.h"

		#include "DeviceEffectManager.h"

		#include "MelReporter.h"

		// Find io handle by session id.
		// Preference is given to an io handle with a matching effect chain to session id.
		// If none found, AUDIO_IO_HANDLE_NONE is returned.
		@@ -1016,6 +1020,7 @@ private:

		const sp<PatchCommandThread> mPatchCommandThread;
		sp<DeviceEffectManager> mDeviceEffectManager;
		sp<MelReporter> mMelReporter;

		bool mSystemReady;
		std::atomic_bool mAudioPolicyReady{};

services/audioflinger/MelReporter.cpp

0 → 100644

+127 −0

Original line number	Diff line number	Diff line
		/*
		**
		** Copyright 2022, The Android Open Source Project
		**
		** Licensed under the Apache License, Version 2.0 (the "License");
		** you may not use this file except in compliance with the License.
		** You may obtain a copy of the License at
		**
		** http://www.apache.org/licenses/LICENSE-2.0
		**
		** Unless required by applicable law or agreed to in writing, software
		** distributed under the License is distributed on an "AS IS" BASIS,
		** WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
		** See the License for the specific language governing permissions and
		** limitations under the License.
		*/

		// #define LOG_NDEBUG 0
		#define LOG_TAG "AudioFlinger::MelReporter"

		#include <cinttypes>
		#include <utils/Log.h>
		#include <android-base/stringprintf.h>
		#include <audio_utils/power.h>

		#include "AudioFlinger.h"

		namespace android {

		bool AudioFlinger::MelReporter::shouldComputeMelForDeviceType(audio_devices_t device) {
		switch (device) {
		case AUDIO_DEVICE_OUT_WIRED_HEADSET:
		case AUDIO_DEVICE_OUT_WIRED_HEADPHONE:
		case AUDIO_DEVICE_OUT_BLUETOOTH_A2DP:
		case AUDIO_DEVICE_OUT_BLUETOOTH_A2DP_HEADPHONES:
		case AUDIO_DEVICE_OUT_HEARING_AID:
		case AUDIO_DEVICE_OUT_USB_HEADSET:
		case AUDIO_DEVICE_OUT_BLE_HEADSET:
		case AUDIO_DEVICE_OUT_BLE_BROADCAST:
		return true;
		default:
		return false;
		}
		}

		void AudioFlinger::MelReporter::onCreateAudioPatch(audio_patch_handle_t handle,
		const PatchPanel::Patch& patch) {
		ALOGV("%s: handle %d mHalHandle %d device sink %08x",
		__func__, handle, patch.mHalHandle,
		patch.mAudioPatch.num_sinks > 0 ? patch.mAudioPatch.sinks[0].ext.device.type : 0);
		if (patch.mAudioPatch.num_sources == 0
		\|\| patch.mAudioPatch.sources[0].type != AUDIO_PORT_TYPE_MIX) {
		ALOGW("%s: patch does not contain any mix sources", __func__);
		return;
		}

		audio_io_handle_t streamHandle = patch.mAudioPatch.sources[0].ext.mix.handle;
		ActiveMelPatch newPatch;
		newPatch.streamHandle = streamHandle;
		for (int i = 0; i < patch.mAudioPatch.num_sinks; ++ i) {
		if (patch.mAudioPatch.sinks[i].type == AUDIO_PORT_TYPE_DEVICE
		&& shouldComputeMelForDeviceType(patch.mAudioPatch.sinks[i].ext.device.type)) {
		audio_port_handle_t deviceId = patch.mAudioPatch.sinks[i].id;
		newPatch.deviceHandles.push_back(deviceId);

		// Start the MEL calculation in the PlaybackThread
		std::lock_guard _lAf(mAudioFlinger.mLock);
		auto thread = mAudioFlinger.checkPlaybackThread_l(streamHandle);
		if (thread != nullptr) {
		thread->startMelComputation(mMelAggregator.getOrCreateCallbackForDevice(
		deviceId,
		newPatch.streamHandle));
		}
		}
		}

		std::lock_guard _l(mLock);
		mActiveMelPatches[patch.mAudioPatch.id] = newPatch;
		}

		void AudioFlinger::MelReporter::onReleaseAudioPatch(audio_patch_handle_t handle) {
		ALOGV("%s", __func__);

		ActiveMelPatch melPatch;
		{
		std::lock_guard _l(mLock);

		auto patchIt = mActiveMelPatches.find(handle);
		if (patchIt == mActiveMelPatches.end()) {
		ALOGW(
		"%s patch does not contain any mix sources with active MEL calculation",
		__func__);
		return;
		}

		melPatch = patchIt->second;
		mActiveMelPatches.erase(patchIt);
		}

		// Stop MEL calculation for the PlaybackThread
		std::lock_guard _lAf(mAudioFlinger.mLock);
		auto thread = mAudioFlinger.checkPlaybackThread_l(melPatch.streamHandle);
		if (thread != nullptr) {
		thread->stopMelComputation();
		}
		mMelAggregator.removeStreamCallback(melPatch.streamHandle);
		}

		std::string AudioFlinger::MelReporter::dump() {
		std::lock_guard _l(mLock);
		std::string output;

		base::StringAppendF(&output, "\nMel Reporter:\n");
		mMelAggregator.foreach([&output](const audio_utils::MelRecord& melRecord) {
		base::StringAppendF(&output, "Continuous MELs for portId=%d, ", melRecord.portId);
		base::StringAppendF(&output, "starting at timestamp %" PRId64 ": ", melRecord.timestamp);

		for (const auto& mel : melRecord.mels) {
		base::StringAppendF(&output, "%d ", mel);
		}
		base::StringAppendF(&output, "\n");
		});

		return output;
		}

		} // namespace android

services/audioflinger/MelReporter.h

0 → 100644

+71 −0

Original line number	Diff line number	Diff line
		/*
		**
		** Copyright 2022, The Android Open Source Project
		**
		** Licensed under the Apache License, Version 2.0 (the "License");
		** you may not use this file except in compliance with the License.
		** You may obtain a copy of the License at
		**
		** http://www.apache.org/licenses/LICENSE-2.0
		**
		** Unless required by applicable law or agreed to in writing, software
		** distributed under the License is distributed on an "AS IS" BASIS,
		** WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
		** See the License for the specific language governing permissions and
		** limitations under the License.
		*/

		#ifndef INCLUDING_FROM_AUDIOFLINGER_H
		#error This header file should only be included from AudioFlinger.h
		#endif

		#include <unordered_map>
		#include <mutex>

		constexpr static int kMaxTimestampDeltaInSec = 120;

		/**
		* Class for listening to new patches and starting the MEL computation. MelReporter is
		* concealed within AudioFlinger, their lifetimes are the same.
		*/
		class MelReporter : public PatchCommandThread::PatchCommandListener {
		public:
		explicit MelReporter(AudioFlinger& audioFlinger)
		: mAudioFlinger(audioFlinger),
		mMelAggregator(kMaxTimestampDeltaInSec) {}

		void onFirstRef() override {
		mAudioFlinger.mPatchCommandThread->addListener(this);
		}

		/** Returns true if we should compute MEL for the given device. */
		static bool shouldComputeMelForDeviceType(audio_devices_t device);

		// For now only support internal MelReporting
		[[nodiscard]] bool isHalReportingEnabled() const { return false; }

		std::string dump();

		// PatchCommandListener methods
		void onCreateAudioPatch(audio_patch_handle_t handle,
		const PatchPanel::Patch& patch) override;
		void onReleaseAudioPatch(audio_patch_handle_t handle) override;

		private:
		AudioFlinger& mAudioFlinger; // does not own the object

		audio_utils::MelAggregator mMelAggregator;

		struct ActiveMelPatch {
		audio_io_handle_t streamHandle{AUDIO_IO_HANDLE_NONE};
		std::vector<audio_port_handle_t> deviceHandles;
		};

		/**
		* Lock for protecting the active mel patches. Do not mix with the AudioFlinger lock.
		* Locking order AudioFlinger::mLock -> PatchCommandThread::mLock -> MelReporter::mLock.
		*/
		std::mutex mLock;
		std::unordered_map<audio_patch_handle_t, ActiveMelPatch>
		mActiveMelPatches GUARDED_BY(AudioFlinger::MelReporter::mLock);
		};