Merge "CSD: Add csd start/stop of BT categorized devices" into udc-qpr-dev (94ef4389) · Commits · e / os / android_frameworks_av

media/libaudioclient/aidl/android/media/ISoundDose.aidl

+24 −0

Original line number	Diff line number	Diff line
		@@ -55,6 +55,30 @@ interface ISoundDose {
		*/
		oneway void setCsdEnabled(boolean enabled);

		/**
		* Structure containing a device identifier by address and type together with
		* the categorization whether it is a headphone or not.
		*/
		@JavaDerive(toString = true)
		parcelable AudioDeviceCategory {
		@utf8InCpp String address;
		int internalAudioType;
		boolean csdCompatible;
		}

		/**
		* Resets the list of stored device categories for the native layer. Should
		* only be called once at boot time after parsing the existing AudioDeviceCategories.
		*/
		oneway void initCachedAudioDeviceCategories(in AudioDeviceCategory[] audioDevices);

		/**
		* Sets whether a device for a given address and type is a headphone or not.
		* This is used to determine whether we compute the CSD on the given device
		* since we can not rely completely on the device annotations.
		*/
		oneway void setAudioDeviceCategory(in AudioDeviceCategory audioDevice);

		/* -------------------------- Test API methods --------------------------
		/** Get the currently used RS2 upper bound. */
		float getOutputRs2UpperBound();

services/audioflinger/MelReporter.cpp

+89 −61

Original line number	Diff line number	Diff line
		@@ -78,30 +78,8 @@ void AudioFlinger::MelReporter::activateInternalSoundDoseComputation() {

		void AudioFlinger::MelReporter::onFirstRef() {
		mAudioFlinger.mPatchCommandThread->addListener(this);
		}

		bool AudioFlinger::MelReporter::shouldComputeMelForDeviceType(audio_devices_t device) {
		if (!mSoundDoseManager->isCsdEnabled()) {
		ALOGV("%s csd is disabled", __func__);
		return false;
		}
		if (mSoundDoseManager->forceComputeCsdOnAllDevices()) {
		return true;
		}

		switch (device) {
		case AUDIO_DEVICE_OUT_WIRED_HEADSET:
		case AUDIO_DEVICE_OUT_WIRED_HEADPHONE:
		// TODO(b/278265907): enable A2DP when we can distinguish A2DP headsets
		// case AUDIO_DEVICE_OUT_BLUETOOTH_A2DP:
		case AUDIO_DEVICE_OUT_BLUETOOTH_A2DP_HEADPHONES:
		case AUDIO_DEVICE_OUT_USB_HEADSET:
		case AUDIO_DEVICE_OUT_BLE_HEADSET:
		case AUDIO_DEVICE_OUT_BLE_BROADCAST:
		return true;
		default:
		return false;
		}
		mSoundDoseManager = sp<SoundDoseManager>::make(sp<IMelReporterCallback>::fromExisting(this));
		}

		void AudioFlinger::MelReporter::updateMetadataForCsd(audio_io_handle_t streamHandle,
		@@ -127,8 +105,8 @@ void AudioFlinger::MelReporter::updateMetadataForCsd(audio_io_handle_t streamHan
		}

		auto activeMelPatchIt = mActiveMelPatches.find(activeMelPatchId.value());
		if (activeMelPatchIt != mActiveMelPatches.end()
		&& shouldActivateCsd != activeMelPatchIt->second.csdActive) {
		if (activeMelPatchIt != mActiveMelPatches.end()) {
		if (shouldActivateCsd != activeMelPatchIt->second.csdActive) {
		if (activeMelPatchIt->second.csdActive) {
		ALOGV("%s should not compute CSD for stream handle %d", __func__, streamHandle);
		stopMelComputationForPatch_l(activeMelPatchIt->second);
		@@ -139,6 +117,7 @@ void AudioFlinger::MelReporter::updateMetadataForCsd(audio_io_handle_t streamHan
		activeMelPatchIt->second.csdActive = shouldActivateCsd;
		}
		}
		}

		void AudioFlinger::MelReporter::onCreateAudioPatch(audio_patch_handle_t handle,
		const PatchPanel::Patch& patch) {
		@@ -159,23 +138,28 @@ void AudioFlinger::MelReporter::onCreateAudioPatch(audio_patch_handle_t handle,
		audio_io_handle_t streamHandle = patch.mAudioPatch.sources[0].ext.mix.handle;
		ActiveMelPatch newPatch;
		newPatch.streamHandle = streamHandle;
		newPatch.csdActive = false;
		for (size_t i = 0; i < patch.mAudioPatch.num_sinks; ++i) {
		if (patch.mAudioPatch.sinks[i].type == AUDIO_PORT_TYPE_DEVICE
		&& shouldComputeMelForDeviceType(patch.mAudioPatch.sinks[i].ext.device.type)) {
		if (patch.mAudioPatch.sinks[i].type == AUDIO_PORT_TYPE_DEVICE &&
		mSoundDoseManager->shouldComputeCsdForDeviceType(
		patch.mAudioPatch.sinks[i].ext.device.type)) {
		audio_port_handle_t deviceId = patch.mAudioPatch.sinks[i].id;
		newPatch.deviceHandles.push_back(deviceId);
		bool shouldComputeCsd = mSoundDoseManager->shouldComputeCsdForDeviceWithAddress(
		patch.mAudioPatch.sinks[i].ext.device.type,
		patch.mAudioPatch.sinks[i].ext.device.address);
		newPatch.deviceStates.push_back({deviceId, shouldComputeCsd});
		newPatch.csdActive \|= shouldComputeCsd;
		AudioDeviceTypeAddr adt{patch.mAudioPatch.sinks[i].ext.device.type,
		patch.mAudioPatch.sinks[i].ext.device.address};
		mSoundDoseManager->mapAddressToDeviceId(adt, deviceId);
		}
		}

		if (!newPatch.deviceHandles.empty()) {
		if (!newPatch.deviceStates.empty() && newPatch.csdActive) {
		std::lock_guard _afl(mAudioFlinger.mLock);
		std::lock_guard _l(mLock);
		ALOGV("%s add patch handle %d to active devices", __func__, handle);
		startMelComputationForActivePatch_l(newPatch);
		newPatch.csdActive = true;
		mActiveMelPatches[handle] = newPatch;
		}
		}
		@@ -189,14 +173,16 @@ NO_THREAD_SAFETY_ANALYSIS // access of AudioFlinger::checkOutputThread_l
		return;
		}

		for (const auto& deviceHandle : patch.deviceHandles) {
		++mActiveDevices[deviceHandle];
		for (const auto& device : patch.deviceStates) {
		if (device.second) {
		++mActiveDevices[device.first];
		ALOGI("%s add stream %d that uses device %d for CSD, nr of streams: %d", __func__,
		patch.streamHandle, deviceHandle, mActiveDevices[deviceHandle]);
		patch.streamHandle, device.first, mActiveDevices[device.first]);

		if (outputThread != nullptr && !useHalSoundDoseInterface_l()) {
		outputThread->startMelComputation_l(mSoundDoseManager->getOrCreateProcessorForDevice(
		deviceHandle,
		outputThread->startMelComputation_l(
		mSoundDoseManager->getOrCreateProcessorForDevice(
		device.first,
		patch.streamHandle,
		outputThread->mSampleRate,
		outputThread->mChannelCount,
		@@ -204,6 +190,27 @@ NO_THREAD_SAFETY_ANALYSIS // access of AudioFlinger::checkOutputThread_l
		}
		}
		}
		}

		void AudioFlinger::MelReporter::startMelComputationForDeviceId(audio_port_handle_t deviceId) {
		ALOGV("%s(%d)", __func__, deviceId);
		std::lock_guard _laf(mAudioFlinger.mLock);
		std::lock_guard _l(mLock);

		for (auto& activeMelPatch : mActiveMelPatches) {
		bool csdActive = false;
		for (auto& device: activeMelPatch.second.deviceStates) {
		if (device.first == deviceId && !device.second) {
		device.second = true;
		}
		csdActive \|= device.second;
		}
		if (csdActive && !activeMelPatch.second.csdActive) {
		activeMelPatch.second.csdActive = csdActive;
		startMelComputationForActivePatch_l(activeMelPatch.second);
		}
		}
		}

		void AudioFlinger::MelReporter::onReleaseAudioPatch(audio_patch_handle_t handle) {
		if (!mSoundDoseManager->isCsdEnabled()) {
		@@ -228,8 +235,12 @@ void AudioFlinger::MelReporter::onReleaseAudioPatch(audio_patch_handle_t handle)

		std::lock_guard _afl(mAudioFlinger.mLock);
		std::lock_guard _l(mLock);
		if (melPatch.csdActive) {
		// only need to stop if patch was active
		melPatch.csdActive = false;
		stopMelComputationForPatch_l(melPatch);
		}
		}

		sp<media::ISoundDose> AudioFlinger::MelReporter::getSoundDoseInterface(
		const sp<media::ISoundDoseCallback>& callback) {
		@@ -247,30 +258,47 @@ void AudioFlinger::MelReporter::stopInternalMelComputation() {
		void AudioFlinger::MelReporter::stopMelComputationForPatch_l(const ActiveMelPatch& patch)
		NO_THREAD_SAFETY_ANALYSIS // access of AudioFlinger::checkOutputThread_l
		{
		if (!patch.csdActive) {
		// no need to stop CSD inactive patches
		return;
		}

		auto outputThread = mAudioFlinger.checkOutputThread_l(patch.streamHandle);

		ALOGV("%s: stop MEL for stream id: %d", __func__, patch.streamHandle);
		for (const auto& deviceId : patch.deviceHandles) {
		if (mActiveDevices[deviceId] > 0) {
		--mActiveDevices[deviceId];
		if (mActiveDevices[deviceId] == 0) {
		for (const auto& device : patch.deviceStates) {
		if (mActiveDevices[device.first] > 0) {
		--mActiveDevices[device.first];
		if (mActiveDevices[device.first] == 0) {
		// no stream is using deviceId anymore
		ALOGI("%s removing device %d from active CSD devices", __func__, deviceId);
		mSoundDoseManager->clearMapDeviceIdEntries(deviceId);
		ALOGI("%s removing device %d from active CSD devices", __func__, device.first);
		mSoundDoseManager->clearMapDeviceIdEntries(device.first);
		}
		}
		}

		mSoundDoseManager->removeStreamProcessor(patch.streamHandle);
		if (outputThread != nullptr && !useHalSoundDoseInterface_l()) {
		outputThread->stopMelComputation_l();
		}
		}

		void AudioFlinger::MelReporter::stopMelComputationForDeviceId(audio_port_handle_t deviceId) {
		ALOGV("%s(%d)", __func__, deviceId);
		std::lock_guard _laf(mAudioFlinger.mLock);
		std::lock_guard _l(mLock);

		for (auto& activeMelPatch : mActiveMelPatches) {
		bool csdActive = false;
		for (auto& device: activeMelPatch.second.deviceStates) {
		if (device.first == deviceId && device.second) {
		device.second = false;
		}
		csdActive \|= device.second;
		}

		if (!csdActive && activeMelPatch.second.csdActive) {
		activeMelPatch.second.csdActive = csdActive;
		stopMelComputationForPatch_l(activeMelPatch.second);
		}
		}

		}

		std::optional<audio_patch_handle_t> AudioFlinger::MelReporter::activePatchStreamHandle_l(
		audio_io_handle_t streamHandle) {

services/audioflinger/MelReporter.h

+13 −7

Original line number	Diff line number	Diff line
		@@ -29,11 +29,11 @@ constexpr static int kMaxTimestampDeltaInSec = 120;
		* Class for listening to new patches and starting the MEL computation. MelReporter is
		* concealed within AudioFlinger, their lifetimes are the same.
		*/
		class MelReporter : public PatchCommandThread::PatchCommandListener {
		class MelReporter : public PatchCommandThread::PatchCommandListener,
		public IMelReporterCallback {
		public:
		explicit MelReporter(AudioFlinger& audioFlinger)
		: mAudioFlinger(audioFlinger),
		mSoundDoseManager(sp<SoundDoseManager>::make()) {}
		: mAudioFlinger(audioFlinger) {}

		void onFirstRef() override;

		@@ -65,6 +65,10 @@ public:

		std::string dump();

		// IMelReporterCallback methods
		void stopMelComputationForDeviceId(audio_port_handle_t deviceId) override;
		void startMelComputationForDeviceId(audio_port_handle_t deviceId) override;

		// PatchCommandListener methods
		void onCreateAudioPatch(audio_patch_handle_t handle,
		const PatchPanel::Patch& patch) override;
		@@ -80,13 +84,15 @@ public:
		private:
		struct ActiveMelPatch {
		audio_io_handle_t streamHandle{AUDIO_IO_HANDLE_NONE};
		std::vector<audio_port_handle_t> deviceHandles;
		/**
		* Stores device ids and whether they are compatible for CSD calculation.
		* The boolean value can change since BT audio device types are user-configurable
		* to headphones/headsets or other device types.
		*/
		std::vector<std::pair<audio_port_handle_t,bool>> deviceStates;
		bool csdActive;
		};

		/** Returns true if we should compute MEL for the given device. */
		bool shouldComputeMelForDeviceType(audio_devices_t device);

		void stopInternalMelComputation();

		/** Should be called with the following order of locks: mAudioFlinger.mLock -> mLock. */

services/audioflinger/sounddose/SoundDoseManager.cpp

+119 −1

Original line number	Diff line number	Diff line
		@@ -299,6 +299,25 @@ binder::Status SoundDoseManager::SoundDose::setCsdEnabled(bool enabled) {
		return binder::Status::ok();
		}

		binder::Status SoundDoseManager::SoundDose::initCachedAudioDeviceCategories(
		const std::vector<media::ISoundDose::AudioDeviceCategory>& btDeviceCategories) {
		ALOGV("%s", __func__);
		auto soundDoseManager = mSoundDoseManager.promote();
		if (soundDoseManager != nullptr) {
		soundDoseManager->initCachedAudioDeviceCategories(btDeviceCategories);
		}
		return binder::Status::ok();
		}
		binder::Status SoundDoseManager::SoundDose::setAudioDeviceCategory(
		const media::ISoundDose::AudioDeviceCategory& btAudioDevice) {
		ALOGV("%s", __func__);
		auto soundDoseManager = mSoundDoseManager.promote();
		if (soundDoseManager != nullptr) {
		soundDoseManager->setAudioDeviceCategory(btAudioDevice);
		}
		return binder::Status::ok();
		}

		binder::Status SoundDoseManager::SoundDose::getOutputRs2UpperBound(float* value) {
		ALOGV("%s", __func__);
		auto soundDoseManager = mSoundDoseManager.promote();
		@@ -356,7 +375,9 @@ void SoundDoseManager::updateAttenuation(float attenuationDB, audio_devices_t de
		auto melProcessor = mp.second.promote();
		if (melProcessor != nullptr) {
		auto deviceId = melProcessor->getDeviceId();
		if (mActiveDeviceTypes[deviceId] == deviceType) {
		const auto deviceTypeIt = mActiveDeviceTypes.find(deviceId);
		if (deviceTypeIt != mActiveDeviceTypes.end() &&
		deviceTypeIt->second == deviceType) {
		ALOGV("%s: set attenuation for deviceId %d to %f",
		__func__, deviceId, attenuationDB);
		melProcessor->setAttenuation(attenuationDB);
		@@ -388,6 +409,103 @@ bool SoundDoseManager::isCsdEnabled() {
		return mEnabledCsd;
		}

		void SoundDoseManager::initCachedAudioDeviceCategories(
		const std::vector<media::ISoundDose::AudioDeviceCategory>& deviceCategories) {
		ALOGV("%s", __func__);
		{
		const std::lock_guard _l(mLock);
		mBluetoothDevicesWithCsd.clear();
		}
		for (const auto& btDeviceCategory : deviceCategories) {
		setAudioDeviceCategory(btDeviceCategory);
		}
		}

		void SoundDoseManager::setAudioDeviceCategory(
		const media::ISoundDose::AudioDeviceCategory& audioDevice) {
		ALOGV("%s: set BT audio device type with address %s to headphone %d", __func__,
		audioDevice.address.c_str(), audioDevice.csdCompatible);

		std::vector<audio_port_handle_t> devicesToStart;
		std::vector<audio_port_handle_t> devicesToStop;
		{
		const std::lock_guard _l(mLock);
		const auto deviceIt = mBluetoothDevicesWithCsd.find(
		std::make_pair(audioDevice.address,
		static_cast<audio_devices_t>(audioDevice.internalAudioType)));
		if (deviceIt != mBluetoothDevicesWithCsd.end()) {
		deviceIt->second = audioDevice.csdCompatible;
		} else {
		mBluetoothDevicesWithCsd.emplace(
		std::make_pair(audioDevice.address,
		static_cast<audio_devices_t>(audioDevice.internalAudioType)),
		audioDevice.csdCompatible);
		}

		for (const auto &activeDevice: mActiveDevices) {
		if (activeDevice.first.address() == audioDevice.address &&
		activeDevice.first.mType ==
		static_cast<audio_devices_t>(audioDevice.internalAudioType)) {
		if (audioDevice.csdCompatible) {
		devicesToStart.push_back(activeDevice.second);
		} else {
		devicesToStop.push_back(activeDevice.second);
		}
		}
		}
		}

		for (const auto& deviceToStart : devicesToStart) {
		mMelReporterCallback->startMelComputationForDeviceId(deviceToStart);
		}
		for (const auto& deviceToStop : devicesToStop) {
		mMelReporterCallback->stopMelComputationForDeviceId(deviceToStop);
		}
		}

		bool SoundDoseManager::shouldComputeCsdForDeviceType(audio_devices_t device) {
		if (!isCsdEnabled()) {
		ALOGV("%s csd is disabled", __func__);
		return false;
		}
		if (forceComputeCsdOnAllDevices()) {
		return true;
		}

		switch (device) {
		case AUDIO_DEVICE_OUT_WIRED_HEADSET:
		case AUDIO_DEVICE_OUT_WIRED_HEADPHONE:
		// TODO(b/278265907): enable A2DP when we can distinguish A2DP headsets
		// case AUDIO_DEVICE_OUT_BLUETOOTH_A2DP:
		case AUDIO_DEVICE_OUT_BLUETOOTH_A2DP_HEADPHONES:
		case AUDIO_DEVICE_OUT_USB_HEADSET:
		case AUDIO_DEVICE_OUT_BLE_HEADSET:
		case AUDIO_DEVICE_OUT_BLE_BROADCAST:
		return true;
		default:
		return false;
		}
		}

		bool SoundDoseManager::shouldComputeCsdForDeviceWithAddress(const audio_devices_t type,
		const std::string& deviceAddress) {
		if (!isCsdEnabled()) {
		ALOGV("%s csd is disabled", __func__);
		return false;
		}
		if (forceComputeCsdOnAllDevices()) {
		return true;
		}

		if (!audio_is_ble_out_device(type) && !audio_is_a2dp_device(type)) {
		return shouldComputeCsdForDeviceType(type);
		}

		const std::lock_guard _l(mLock);
		const auto deviceIt = mBluetoothDevicesWithCsd.find(std::make_pair(deviceAddress, type));
		return deviceIt != mBluetoothDevicesWithCsd.end() && deviceIt->second;
		}

		void SoundDoseManager::setUseFrameworkMel(bool useFrameworkMel) {
		// invalidate any HAL sound dose interface used
		setHalSoundDoseInterface(nullptr);

services/audioflinger/sounddose/SoundDoseManager.h

+47 −2

Original line number	Diff line number	Diff line
		@@ -32,6 +32,15 @@ namespace android {

		using aidl::android::hardware::audio::core::sounddose::ISoundDose;

		class IMelReporterCallback : public virtual RefBase {
		public:
		IMelReporterCallback() {};
		virtual ~IMelReporterCallback() {};

		virtual void stopMelComputationForDeviceId(audio_port_handle_t deviceId) = 0;
		virtual void startMelComputationForDeviceId(audio_port_handle_t deviceId) = 0;
		};

		class SoundDoseManager : public audio_utils::MelProcessor::MelCallback {
		public:
		/** CSD is computed with a rolling window of 7 days. */
		@@ -39,8 +48,9 @@ public:
		/** Default RS2 upper bound in dBA as defined in IEC 62368-1 3rd edition. */
		static constexpr float kDefaultRs2UpperBound = 100.f;

		SoundDoseManager()
		: mMelAggregator(sp<audio_utils::MelAggregator>::make(kCsdWindowSeconds)),
		explicit SoundDoseManager(const sp<IMelReporterCallback>& melReporterCallback)
		: mMelReporterCallback(melReporterCallback),
		mMelAggregator(sp<audio_utils::MelAggregator>::make(kCsdWindowSeconds)),
		mRs2UpperBound(kDefaultRs2UpperBound) {};

		/**
		@@ -104,6 +114,21 @@ public:
		/** Returns true if CSD is enabled. */
		bool isCsdEnabled();

		void initCachedAudioDeviceCategories(
		const std::vector<media::ISoundDose::AudioDeviceCategory>& deviceCategories);

		void setAudioDeviceCategory(
		const media::ISoundDose::AudioDeviceCategory& audioDevice);

		/**
		* Returns true if the type can compute CSD. For bluetooth devices we rely on whether we
		* categorized the address as headphones/headsets, only in this case we return true.
		*/
		bool shouldComputeCsdForDeviceWithAddress(const audio_devices_t type,
		const std::string& deviceAddress);
		/** Returns true for all device types which could support CSD computation. */
		bool shouldComputeCsdForDeviceType(audio_devices_t device);

		std::string dump() const;

		// used for testing only
		@@ -139,6 +164,13 @@ private:
		binder::Status getOutputRs2UpperBound(float* value) override;
		binder::Status setCsdEnabled(bool enabled) override;

		binder::Status initCachedAudioDeviceCategories(
		const std::vector<media::ISoundDose::AudioDeviceCategory> &btDeviceCategories)
		override;

		binder::Status setAudioDeviceCategory(
		const media::ISoundDose::AudioDeviceCategory& btAudioDevice) override;

		binder::Status getCsd(float* value) override;
		binder::Status forceUseFrameworkMel(bool useFrameworkMel) override;
		binder::Status forceComputeCsdOnAllDevices(bool computeCsdOnAllDevices) override;
		@@ -179,6 +211,8 @@ private:

		mutable std::mutex mLock;

		const sp<IMelReporterCallback> mMelReporterCallback;

		// no need for lock since MelAggregator is thread-safe
		const sp<audio_utils::MelAggregator> mMelAggregator;

		@@ -191,6 +225,17 @@ private:
		std::map<AudioDeviceTypeAddr, audio_port_handle_t> mActiveDevices GUARDED_BY(mLock);
		std::unordered_map<audio_port_handle_t, audio_devices_t> mActiveDeviceTypes GUARDED_BY(mLock);

		struct bt_device_type_hash {
		std::size_t operator() (const std::pair<std::string, audio_devices_t> &deviceType) const {
		return std::hash<std::string>()(deviceType.first) ^
		std::hash<audio_devices_t>()(deviceType.second);
		}
		};
		// storing the BT cached information as received from the java side
		// see SoundDoseManager::setCachedAudioDeviceCategories
		std::unordered_map<std::pair<std::string, audio_devices_t>, bool, bt_device_type_hash>
		mBluetoothDevicesWithCsd GUARDED_BY(mLock);

		float mRs2UpperBound GUARDED_BY(mLock);
		std::unordered_map<audio_devices_t, float> mMelAttenuationDB GUARDED_BY(mLock);