Spatial Audio: Generalize VectorRecorder for logging am: a367cb210b am: c4eeeb7e52 (85a03cea) · Commits · e / os / android_frameworks_av

media/libheadtracking/Android.bp

+4 −0

Original line number	Original line	Diff line number	Diff line
	@@ -21,6 +21,7 @@ cc_library {
	"ScreenHeadFusion.cpp",		"ScreenHeadFusion.cpp",
	"StillnessDetector.cpp",		"StillnessDetector.cpp",
	"Twist.cpp",		"Twist.cpp",
			"VectorRecorder.cpp",
	],		],
	shared_libs: [		shared_libs: [
	"libaudioutils",		"libaudioutils",
	@@ -35,6 +36,9 @@ cc_library {
	export_header_lib_headers: [		export_header_lib_headers: [
	"libeigen",		"libeigen",
	],		],
			cflags: [
			"-Wthread-safety",
			],
	}		}

	cc_library {		cc_library {

media/libheadtracking/VectorRecorder.cpp

0 → 100644

+88 −0

Original line number	Original line	Diff line number	Diff line
			/*
			* Copyright (C) 2023 The Android Open Source Project
			*
			* Licensed under the Apache License, Version 2.0 (the "License");
			* you may not use this file except in compliance with the License.
			* You may obtain a copy of the License at
			*
			* http://www.apache.org/licenses/LICENSE-2.0
			*
			* Unless required by applicable law or agreed to in writing, software
			* distributed under the License is distributed on an "AS IS" BASIS,
			* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
			* See the License for the specific language governing permissions and
			* limitations under the License.
			*/

			#include "media/VectorRecorder.h"

			namespace android::media {

			// Convert data to string with level indentation.
			// No need for a lock as the SimpleLog is thread-safe.
			std::string VectorRecorder::toString(size_t indent) const {
			return mRecordLog.dumpToString(std::string(indent + 1, ' ').c_str(), mMaxLocalLogLine);
			}

			// Record into local log when it is time.
			void VectorRecorder::record(const std::vector<float>& record) {
			if (record.size() != mVectorSize) return;

			// Protect against concurrent calls to record().
			std::lock_guard lg(mLock);

			// if it is time, record average data and reset.
			if (shouldRecordLog_l()) {
			sumToAverage_l();
			mRecordLog.log(
			"mean: %s, min: %s, max %s, calculated %zu samples in %0.4f second(s)",
			toString(mSum).c_str(),
			toString(mMin).c_str(),
			toString(mMax).c_str(),
			mNumberOfSamples,
			mNumberOfSecondsSinceFirstSample.count());
			resetRecord_l();
			}

			// update stream average.
			if (mNumberOfSamples++ == 0) {
			mFirstSampleTimestamp = std::chrono::steady_clock::now();
			for (size_t i = 0; i < mVectorSize; ++i) {
			const float value = record[i];
			mSum[i] += value;
			mMax[i] = value;
			mMin[i] = value;
			}
			} else {
			for (size_t i = 0; i < mVectorSize; ++i) {
			const float value = record[i];
			mSum[i] += value;
			mMax[i] = std::max(mMax[i], value);
			mMin[i] = std::min(mMin[i], value);
			}
			}
			}

			bool VectorRecorder::shouldRecordLog_l() {
			mNumberOfSecondsSinceFirstSample = std::chrono::duration_cast<std::chrono::seconds>(
			std::chrono::steady_clock::now() - mFirstSampleTimestamp);
			return mNumberOfSecondsSinceFirstSample >= mRecordThreshold;
			}

			void VectorRecorder::resetRecord_l() {
			mSum.assign(mVectorSize, 0);
			mMax.assign(mVectorSize, 0);
			mMin.assign(mVectorSize, 0);
			mNumberOfSamples = 0;
			mNumberOfSecondsSinceFirstSample = std::chrono::seconds(0);
			}

			void VectorRecorder::sumToAverage_l() {
			if (mNumberOfSamples == 0) return;
			const float reciprocal = 1.f / mNumberOfSamples;
			for (auto& p : mSum) {
			p *= reciprocal;
			}
			}

			} // namespace android::media

media/libheadtracking/include/media/VectorRecorder.h

0 → 100644

+114 −0

Original line number	Original line	Diff line number	Diff line
			/*
			* Copyright (C) 2023 The Android Open Source Project
			*
			* Licensed under the Apache License, Version 2.0 (the "License");
			* you may not use this file except in compliance with the License.
			* You may obtain a copy of the License at
			*
			* http://www.apache.org/licenses/LICENSE-2.0
			*
			* Unless required by applicable law or agreed to in writing, software
			* distributed under the License is distributed on an "AS IS" BASIS,
			* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
			* See the License for the specific language governing permissions and
			* limitations under the License.
			*/

			#pragma once

			#include <android-base/stringprintf.h>
			#include <android-base/thread_annotations.h>
			#include <audio_utils/SimpleLog.h>
			#include <chrono>
			#include <math.h>
			#include <mutex>
			#include <vector>

			namespace android::media {

			/**
			* VectorRecorder records a vector of floats computing the average, max, and min
			* over given time periods.
			*
			* The class is thread-safe.
			*/
			class VectorRecorder {
			public:
			VectorRecorder(
			size_t vectorSize, std::chrono::duration<double> threshold, int maxLogLine)
			: mVectorSize(vectorSize)
			, mRecordLog(maxLogLine)
			, mRecordThreshold(threshold)
			{
			resetRecord_l(); // OK to call - we're in the constructor.
			}

			/** Convert recorded vector data to string with level indentation */
			std::string toString(size_t indent) const;

			/**
			* @brief Record a vector of floats.
			*
			* @param record a vector of floats.
			*/
			void record(const std::vector<float>& record);

			/**
			* Format vector to a string, [0.00, 0.00, 0.00, -1.29, -0.50, 15.27].
			*/
			template <typename T>
			static std::string toString(const std::vector<T>& record) {
			if (record.size() == 0) {
			return "[]";
			}

			std::string ss = "[";
			for (size_t i = 0; i < record.size(); ++i) {
			if (i > 0) {
			ss.append(", ");
			}
			base::StringAppendF(&ss, "%0.2lf", static_cast<double>(record[i]));
			}
			ss.append("]");
			return ss;
			}

			private:
			static constexpr int mMaxLocalLogLine = 10;

			const size_t mVectorSize;

			// Local log for historical vector data.
			// Locked internally, so does not need mutex below.
			SimpleLog mRecordLog{mMaxLocalLogLine};

			std::mutex mLock;

			// Time threshold to record vectors in the local log.
			// Vector data will be recorded into log at least every mRecordThreshold.
			std::chrono::duration<double> mRecordThreshold GUARDED_BY(mLock);

			// Number of seconds since first sample in mSum.
			std::chrono::duration<double> mNumberOfSecondsSinceFirstSample GUARDED_BY(mLock);

			// Timestamp of first sample recorded in mSum.
			std::chrono::time_point<std::chrono::steady_clock> mFirstSampleTimestamp GUARDED_BY(mLock);

			// Number of samples in mSum.
			size_t mNumberOfSamples GUARDED_BY(mLock) = 0;

			std::vector<double> mSum GUARDED_BY(mLock);
			std::vector<float> mMax GUARDED_BY(mLock);
			std::vector<float> mMin GUARDED_BY(mLock);

			// Computes mNumberOfSecondsSinceFirstSample, returns true if time to record.
			bool shouldRecordLog_l() REQUIRES(mLock);

			// Resets the running mNumberOfSamples, mSum, mMax, mMin.
			void resetRecord_l() REQUIRES(mLock);

			// Convert mSum to an average.
			void sumToAverage_l() REQUIRES(mLock);
			}; // VectorRecorder

			} // namespace android::media

services/audiopolicy/service/Spatializer.cpp

+12 −36

Original line number	Original line	Diff line number	Diff line
	@@ -75,6 +75,16 @@ static audio_channel_mask_t getMaxChannelMask(
	return maxMask;		return maxMask;
	}		}

			std::vector<float> recordFromRotationVector(const std::vector<float>& rotationVector) {
			constexpr float RAD_TO_DEGREE = 180.f / M_PI;
			std::vector<float> record{
			rotationVector[0], rotationVector[1], rotationVector[2],
			rotationVector[3] * RAD_TO_DEGREE,
			rotationVector[4] * RAD_TO_DEGREE,
			rotationVector[5] * RAD_TO_DEGREE};
			return record;
			}

	// ---------------------------------------------------------------------------		// ---------------------------------------------------------------------------

	class Spatializer::EngineCallbackHandler : public AHandler {		class Spatializer::EngineCallbackHandler : public AHandler {
	@@ -184,41 +194,6 @@ const std::vector<const char *> Spatializer::sHeadPoseKeys = {
	Spatializer::EngineCallbackHandler::kRotation2Key,		Spatializer::EngineCallbackHandler::kRotation2Key,
	};		};

	// ---------------------------------------------------------------------------

	// Convert recorded sensor data to string with level indentation.
	std::string Spatializer::HeadToStagePoseRecorder::toString(unsigned level) const {
	std::string prefixSpace(level, ' ');
	return mPoseRecordLog.dumpToString((prefixSpace + " ").c_str(), Spatializer::mMaxLocalLogLine);
	}

	// Compute sensor data, record into local log when it is time.
	void Spatializer::HeadToStagePoseRecorder::record(const std::vector<float>& headToStage) {
	if (headToStage.size() != mPoseVectorSize) return;

	if (mNumOfSampleSinceLastRecord++ == 0) {
	mFirstSampleTimestamp = std::chrono::steady_clock::now();
	}
	// if it's time, do record and reset.
	if (shouldRecordLog()) {
	poseSumToAverage();
	mPoseRecordLog.log(
	"mean: %s, min: %s, max %s, calculated %d samples in %0.4f second(s)",
	Spatializer::toString<double>(mPoseRadianSum, true /* radianToDegree */).c_str(),
	Spatializer::toString<float>(mMinPoseAngle, true /* radianToDegree */).c_str(),
	Spatializer::toString<float>(mMaxPoseAngle, true /* radianToDegree */).c_str(),
	mNumOfSampleSinceLastRecord, mNumOfSecondsSinceLastRecord.count());
	resetRecord();
	}
	// update stream average.
	for (int i = 0; i < mPoseVectorSize; i++) {
	mPoseRadianSum[i] += headToStage[i];
	mMaxPoseAngle[i] = std::max(mMaxPoseAngle[i], headToStage[i]);
	mMinPoseAngle[i] = std::min(mMinPoseAngle[i], headToStage[i]);
	}
	return;
	}

	// ---------------------------------------------------------------------------		// ---------------------------------------------------------------------------
	sp<Spatializer> Spatializer::create(SpatializerPolicyCallback *callback) {		sp<Spatializer> Spatializer::create(SpatializerPolicyCallback *callback) {
	sp<Spatializer> spatializer;		sp<Spatializer> spatializer;
	@@ -590,7 +565,8 @@ Status Spatializer::setGlobalTransform(const std::vector<float>& screenToStage)
	}		}
	std::lock_guard lock(mLock);		std::lock_guard lock(mLock);
	if (mPoseController != nullptr) {		if (mPoseController != nullptr) {
	mLocalLog.log("%s with screenToStage %s", __func__, toString<float>(screenToStage).c_str());		mLocalLog.log("%s with screenToStage %s", __func__,
			media::VectorRecorder::toString<float>(screenToStage).c_str());
	mPoseController->setScreenToStagePose(maybePose.value());		mPoseController->setScreenToStagePose(maybePose.value());
	}		}
	return Status::ok();		return Status::ok();

services/audiopolicy/service/Spatializer.h

+5 −108

Original line number	Original line	Diff line number	Diff line
	@@ -27,6 +27,7 @@
	#include <audio_utils/SimpleLog.h>		#include <audio_utils/SimpleLog.h>
	#include <math.h>		#include <math.h>
	#include <media/AudioEffect.h>		#include <media/AudioEffect.h>
			#include <media/VectorRecorder.h>
	#include <media/audiohal/EffectHalInterface.h>		#include <media/audiohal/EffectHalInterface.h>
	#include <media/stagefright/foundation/ALooper.h>		#include <media/stagefright/foundation/ALooper.h>
	#include <system/audio_effects/effect_spatializer.h>		#include <system/audio_effects/effect_spatializer.h>
	@@ -173,30 +174,6 @@ class Spatializer : public media::BnSpatializer,
	media::audio::common::toString(*result) : "unknown_latency_mode";		media::audio::common::toString(*result) : "unknown_latency_mode";
	}		}

	/**
	* Format head to stage vector to a string, [0.00, 0.00, 0.00, -1.29, -0.50, 15.27].
	*/
	template <typename T>
	static std::string toString(const std::vector<T>& vec, bool radianToDegree = false) {
	if (vec.size() == 0) {
	return "[]";
	}

	std::string ss = "[";
	for (auto f = vec.begin(); f != vec.end(); ++f) {
	if (f != vec.begin()) {
	ss .append(", ");
	}
	if (radianToDegree) {
	base::StringAppendF(&ss, "%0.2f", HeadToStagePoseRecorder::getDegreeWithRadian(*f));
	} else {
	base::StringAppendF(&ss, "%f", *f);
	}
	}
	ss.append("]");
	return ss;
	};

	// If the Spatializer is not created, we send the status for metrics purposes.		// If the Spatializer is not created, we send the status for metrics purposes.
	// OK: Spatializer not expected to be created.		// OK: Spatializer not expected to be created.
	// NO_INIT: Spatializer creation failed.		// NO_INIT: Spatializer creation failed.
	@@ -428,92 +405,12 @@ private:
	* @brief Calculate and record sensor data.		* @brief Calculate and record sensor data.
	* Dump to local log with max/average pose angle every mPoseRecordThreshold.		* Dump to local log with max/average pose angle every mPoseRecordThreshold.
	*/		*/
	class HeadToStagePoseRecorder {
	public:
	HeadToStagePoseRecorder(std::chrono::duration<double> threshold, int maxLogLine)
	: mPoseRecordThreshold(threshold), mPoseRecordLog(maxLogLine) {
	resetRecord();
	}

	/** Convert recorded sensor data to string with level indentation */
	std::string toString(unsigned level) const;

	/**
	* @brief Calculate sensor data, record into local log when it is time.
	*
	* @param headToStage The vector from Pose3f::toVector().
	*/
	void record(const std::vector<float>& headToStage);

	static constexpr float getDegreeWithRadian(const float radian) {
	float radianToDegreeRatio = (180 / PI);
	return (radian * radianToDegreeRatio);
	}

	private:
	static constexpr float PI = M_PI;
	/**
	* Pose recorder time threshold to record sensor data in local log.
	* Sensor data will be recorded into log at least every mPoseRecordThreshold.
	*/
	std::chrono::duration<double> mPoseRecordThreshold;
	// Number of seconds pass since last record.
	std::chrono::duration<double> mNumOfSecondsSinceLastRecord;
	/**
	* According to frameworks/av/media/libheadtracking/include/media/Pose.h
	* "The vector will have exactly 6 elements, where the first three are a translation vector
	* and the last three are a rotation vector."
	*/
	static constexpr size_t mPoseVectorSize = 6;
	/**
	* Timestamp of last sensor data record in local log.
	*/
	std::chrono::time_point<std::chrono::steady_clock> mFirstSampleTimestamp;
	/**
	* Number of sensor samples received since last record, sample rate is ~100Hz which produce
	* ~6k samples/minute.
	*/
	uint32_t mNumOfSampleSinceLastRecord = 0;
	/* The sum of pose angle represented by radian since last dump, div
	* mNumOfSampleSinceLastRecord to get arithmetic mean. Largest possible value: 2PI * 100Hz *
	* mPoseRecordThreshold.
	*/
	std::vector<double> mPoseRadianSum;
	std::vector<float> mMaxPoseAngle;
	std::vector<float> mMinPoseAngle;
	// Local log for history sensor data.
	SimpleLog mPoseRecordLog{mMaxLocalLogLine};

	bool shouldRecordLog() {
	mNumOfSecondsSinceLastRecord = std::chrono::duration_cast<std::chrono::seconds>(
	std::chrono::steady_clock::now() - mFirstSampleTimestamp);
	return mNumOfSecondsSinceLastRecord >= mPoseRecordThreshold;
	}

	void resetRecord() {
	mPoseRadianSum.assign(mPoseVectorSize, 0);
	mMaxPoseAngle.assign(mPoseVectorSize, -PI);
	mMinPoseAngle.assign(mPoseVectorSize, PI);
	mNumOfSampleSinceLastRecord = 0;
	mNumOfSecondsSinceLastRecord = std::chrono::seconds(0);
	}

	// Add each sample to sum and only calculate when record.
	void poseSumToAverage() {
	if (mNumOfSampleSinceLastRecord == 0) return;
	for (auto& p : mPoseRadianSum) {
	const float reciprocal = 1.f / mNumOfSampleSinceLastRecord;
	p *= reciprocal;
	}
	}
	}; // HeadToStagePoseRecorder

	// Record one log line per second (up to mMaxLocalLogLine) to capture most recent sensor data.		// Record one log line per second (up to mMaxLocalLogLine) to capture most recent sensor data.
	HeadToStagePoseRecorder mPoseRecorder GUARDED_BY(mLock) =		media::VectorRecorder mPoseRecorder GUARDED_BY(mLock) {
	HeadToStagePoseRecorder(std::chrono::seconds(1), mMaxLocalLogLine);		6 /* vectorSize */, std::chrono::seconds(1), mMaxLocalLogLine };
	// Record one log line per minute (up to mMaxLocalLogLine) to capture durable sensor data.		// Record one log line per minute (up to mMaxLocalLogLine) to capture durable sensor data.
	HeadToStagePoseRecorder mPoseDurableRecorder GUARDED_BY(mLock) =		media::VectorRecorder mPoseDurableRecorder GUARDED_BY(mLock) {
	HeadToStagePoseRecorder(std::chrono::minutes(1), mMaxLocalLogLine);		6 /* vectorSize */, std::chrono::minutes(1), mMaxLocalLogLine };
	}; // Spatializer		}; // Spatializer

	}; // namespace android		}; // namespace android