Spatial Audio: Generalize VectorRecorder for logging (82a4eab1) · Commits · e / os / android_frameworks_av

media/libheadtracking/Android.bp

+4 −0

Original line number	Diff line number	Diff line
		@@ -21,6 +21,7 @@ cc_library {
		"ScreenHeadFusion.cpp",
		"StillnessDetector.cpp",
		"Twist.cpp",
		"VectorRecorder.cpp",
		],
		shared_libs: [
		"libaudioutils",
		@@ -35,6 +36,9 @@ cc_library {
		export_header_lib_headers: [
		"libeigen",
		],
		cflags: [
		"-Wthread-safety",
		],
		}

		cc_library {

media/libheadtracking/VectorRecorder.cpp

0 → 100644

+88 −0

Original line number	Diff line number	Diff line
		/*
		* Copyright (C) 2023 The Android Open Source Project
		*
		* Licensed under the Apache License, Version 2.0 (the "License");
		* you may not use this file except in compliance with the License.
		* You may obtain a copy of the License at
		*
		* http://www.apache.org/licenses/LICENSE-2.0
		*
		* Unless required by applicable law or agreed to in writing, software
		* distributed under the License is distributed on an "AS IS" BASIS,
		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
		* See the License for the specific language governing permissions and
		* limitations under the License.
		*/

		#include "media/VectorRecorder.h"

		namespace android::media {

		// Convert data to string with level indentation.
		// No need for a lock as the SimpleLog is thread-safe.
		std::string VectorRecorder::toString(size_t indent) const {
		return mRecordLog.dumpToString(std::string(indent + 1, ' ').c_str(), mMaxLocalLogLine);
		}

		// Record into local log when it is time.
		void VectorRecorder::record(const std::vector<float>& record) {
		if (record.size() != mVectorSize) return;

		// Protect against concurrent calls to record().
		std::lock_guard lg(mLock);

		// if it is time, record average data and reset.
		if (shouldRecordLog_l()) {
		sumToAverage_l();
		mRecordLog.log(
		"mean: %s, min: %s, max %s, calculated %zu samples in %0.4f second(s)",
		toString(mSum).c_str(),
		toString(mMin).c_str(),
		toString(mMax).c_str(),
		mNumberOfSamples,
		mNumberOfSecondsSinceFirstSample.count());
		resetRecord_l();
		}

		// update stream average.
		if (mNumberOfSamples++ == 0) {
		mFirstSampleTimestamp = std::chrono::steady_clock::now();
		for (size_t i = 0; i < mVectorSize; ++i) {
		const float value = record[i];
		mSum[i] += value;
		mMax[i] = value;
		mMin[i] = value;
		}
		} else {
		for (size_t i = 0; i < mVectorSize; ++i) {
		const float value = record[i];
		mSum[i] += value;
		mMax[i] = std::max(mMax[i], value);
		mMin[i] = std::min(mMin[i], value);
		}
		}
		}

		bool VectorRecorder::shouldRecordLog_l() {
		mNumberOfSecondsSinceFirstSample = std::chrono::duration_cast<std::chrono::seconds>(
		std::chrono::steady_clock::now() - mFirstSampleTimestamp);
		return mNumberOfSecondsSinceFirstSample >= mRecordThreshold;
		}

		void VectorRecorder::resetRecord_l() {
		mSum.assign(mVectorSize, 0);
		mMax.assign(mVectorSize, 0);
		mMin.assign(mVectorSize, 0);
		mNumberOfSamples = 0;
		mNumberOfSecondsSinceFirstSample = std::chrono::seconds(0);
		}

		void VectorRecorder::sumToAverage_l() {
		if (mNumberOfSamples == 0) return;
		const float reciprocal = 1.f / mNumberOfSamples;
		for (auto& p : mSum) {
		p *= reciprocal;
		}
		}

		} // namespace android::media

media/libheadtracking/include/media/VectorRecorder.h

0 → 100644

+114 −0

Original line number	Diff line number	Diff line
		/*
		* Copyright (C) 2023 The Android Open Source Project
		*
		* Licensed under the Apache License, Version 2.0 (the "License");
		* you may not use this file except in compliance with the License.
		* You may obtain a copy of the License at
		*
		* http://www.apache.org/licenses/LICENSE-2.0
		*
		* Unless required by applicable law or agreed to in writing, software
		* distributed under the License is distributed on an "AS IS" BASIS,
		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
		* See the License for the specific language governing permissions and
		* limitations under the License.
		*/

		#pragma once

		#include <android-base/stringprintf.h>
		#include <android-base/thread_annotations.h>
		#include <audio_utils/SimpleLog.h>
		#include <chrono>
		#include <math.h>
		#include <mutex>
		#include <vector>

		namespace android::media {

		/**
		* VectorRecorder records a vector of floats computing the average, max, and min
		* over given time periods.
		*
		* The class is thread-safe.
		*/
		class VectorRecorder {
		public:
		VectorRecorder(
		size_t vectorSize, std::chrono::duration<double> threshold, int maxLogLine)
		: mVectorSize(vectorSize)
		, mRecordLog(maxLogLine)
		, mRecordThreshold(threshold)
		{
		resetRecord_l(); // OK to call - we're in the constructor.
		}

		/** Convert recorded vector data to string with level indentation */
		std::string toString(size_t indent) const;

		/**
		* @brief Record a vector of floats.
		*
		* @param record a vector of floats.
		*/
		void record(const std::vector<float>& record);

		/**
		* Format vector to a string, [0.00, 0.00, 0.00, -1.29, -0.50, 15.27].
		*/
		template <typename T>
		static std::string toString(const std::vector<T>& record) {
		if (record.size() == 0) {
		return "[]";
		}

		std::string ss = "[";
		for (size_t i = 0; i < record.size(); ++i) {
		if (i > 0) {
		ss.append(", ");
		}
		base::StringAppendF(&ss, "%0.2lf", static_cast<double>(record[i]));
		}
		ss.append("]");
		return ss;
		}

		private:
		static constexpr int mMaxLocalLogLine = 10;

		const size_t mVectorSize;

		// Local log for historical vector data.
		// Locked internally, so does not need mutex below.
		SimpleLog mRecordLog{mMaxLocalLogLine};

		std::mutex mLock;

		// Time threshold to record vectors in the local log.
		// Vector data will be recorded into log at least every mRecordThreshold.
		std::chrono::duration<double> mRecordThreshold GUARDED_BY(mLock);

		// Number of seconds since first sample in mSum.
		std::chrono::duration<double> mNumberOfSecondsSinceFirstSample GUARDED_BY(mLock);

		// Timestamp of first sample recorded in mSum.
		std::chrono::time_point<std::chrono::steady_clock> mFirstSampleTimestamp GUARDED_BY(mLock);

		// Number of samples in mSum.
		size_t mNumberOfSamples GUARDED_BY(mLock) = 0;

		std::vector<double> mSum GUARDED_BY(mLock);
		std::vector<float> mMax GUARDED_BY(mLock);
		std::vector<float> mMin GUARDED_BY(mLock);

		// Computes mNumberOfSecondsSinceFirstSample, returns true if time to record.
		bool shouldRecordLog_l() REQUIRES(mLock);

		// Resets the running mNumberOfSamples, mSum, mMax, mMin.
		void resetRecord_l() REQUIRES(mLock);

		// Convert mSum to an average.
		void sumToAverage_l() REQUIRES(mLock);
		}; // VectorRecorder

		} // namespace android::media

services/audiopolicy/service/Spatializer.cpp

+12 −36

Original line number	Diff line number	Diff line
		@@ -75,6 +75,16 @@ static audio_channel_mask_t getMaxChannelMask(
		return maxMask;
		}

		std::vector<float> recordFromRotationVector(const std::vector<float>& rotationVector) {
		constexpr float RAD_TO_DEGREE = 180.f / M_PI;
		std::vector<float> record{
		rotationVector[0], rotationVector[1], rotationVector[2],
		rotationVector[3] * RAD_TO_DEGREE,
		rotationVector[4] * RAD_TO_DEGREE,
		rotationVector[5] * RAD_TO_DEGREE};
		return record;
		}

		// ---------------------------------------------------------------------------

		class Spatializer::EngineCallbackHandler : public AHandler {
		@@ -184,41 +194,6 @@ const std::vector<const char *> Spatializer::sHeadPoseKeys = {
		Spatializer::EngineCallbackHandler::kRotation2Key,
		};

		// ---------------------------------------------------------------------------

		// Convert recorded sensor data to string with level indentation.
		std::string Spatializer::HeadToStagePoseRecorder::toString(unsigned level) const {
		std::string prefixSpace(level, ' ');
		return mPoseRecordLog.dumpToString((prefixSpace + " ").c_str(), Spatializer::mMaxLocalLogLine);
		}

		// Compute sensor data, record into local log when it is time.
		void Spatializer::HeadToStagePoseRecorder::record(const std::vector<float>& headToStage) {
		if (headToStage.size() != mPoseVectorSize) return;

		if (mNumOfSampleSinceLastRecord++ == 0) {
		mFirstSampleTimestamp = std::chrono::steady_clock::now();
		}
		// if it's time, do record and reset.
		if (shouldRecordLog()) {
		poseSumToAverage();
		mPoseRecordLog.log(
		"mean: %s, min: %s, max %s, calculated %d samples in %0.4f second(s)",
		Spatializer::toString<double>(mPoseRadianSum, true /* radianToDegree */).c_str(),
		Spatializer::toString<float>(mMinPoseAngle, true /* radianToDegree */).c_str(),
		Spatializer::toString<float>(mMaxPoseAngle, true /* radianToDegree */).c_str(),
		mNumOfSampleSinceLastRecord, mNumOfSecondsSinceLastRecord.count());
		resetRecord();
		}
		// update stream average.
		for (int i = 0; i < mPoseVectorSize; i++) {
		mPoseRadianSum[i] += headToStage[i];
		mMaxPoseAngle[i] = std::max(mMaxPoseAngle[i], headToStage[i]);
		mMinPoseAngle[i] = std::min(mMinPoseAngle[i], headToStage[i]);
		}
		return;
		}

		// ---------------------------------------------------------------------------
		sp<Spatializer> Spatializer::create(SpatializerPolicyCallback *callback) {
		sp<Spatializer> spatializer;
		@@ -590,7 +565,8 @@ Status Spatializer::setGlobalTransform(const std::vector<float>& screenToStage)
		}
		std::lock_guard lock(mLock);
		if (mPoseController != nullptr) {
		mLocalLog.log("%s with screenToStage %s", __func__, toString<float>(screenToStage).c_str());
		mLocalLog.log("%s with screenToStage %s", __func__,
		media::VectorRecorder::toString<float>(screenToStage).c_str());
		mPoseController->setScreenToStagePose(maybePose.value());
		}
		return Status::ok();

services/audiopolicy/service/Spatializer.h

+5 −108

Original line number	Diff line number	Diff line
		@@ -27,6 +27,7 @@
		#include <audio_utils/SimpleLog.h>
		#include <math.h>
		#include <media/AudioEffect.h>
		#include <media/VectorRecorder.h>
		#include <media/audiohal/EffectHalInterface.h>
		#include <media/stagefright/foundation/ALooper.h>
		#include <system/audio_effects/effect_spatializer.h>
		@@ -172,30 +173,6 @@ class Spatializer : public media::BnSpatializer,
		media::audio::common::toString(*result) : "unknown_latency_mode";
		}

		/**
		* Format head to stage vector to a string, [0.00, 0.00, 0.00, -1.29, -0.50, 15.27].
		*/
		template <typename T>
		static std::string toString(const std::vector<T>& vec, bool radianToDegree = false) {
		if (vec.size() == 0) {
		return "[]";
		}

		std::string ss = "[";
		for (auto f = vec.begin(); f != vec.end(); ++f) {
		if (f != vec.begin()) {
		ss .append(", ");
		}
		if (radianToDegree) {
		base::StringAppendF(&ss, "%0.2f", HeadToStagePoseRecorder::getDegreeWithRadian(*f));
		} else {
		base::StringAppendF(&ss, "%f", *f);
		}
		}
		ss.append("]");
		return ss;
		};

		// If the Spatializer is not created, we send the status for metrics purposes.
		// OK: Spatializer not expected to be created.
		// NO_INIT: Spatializer creation failed.
		@@ -427,92 +404,12 @@ private:
		* @brief Calculate and record sensor data.
		* Dump to local log with max/average pose angle every mPoseRecordThreshold.
		*/
		class HeadToStagePoseRecorder {
		public:
		HeadToStagePoseRecorder(std::chrono::duration<double> threshold, int maxLogLine)
		: mPoseRecordThreshold(threshold), mPoseRecordLog(maxLogLine) {
		resetRecord();
		}

		/** Convert recorded sensor data to string with level indentation */
		std::string toString(unsigned level) const;

		/**
		* @brief Calculate sensor data, record into local log when it is time.
		*
		* @param headToStage The vector from Pose3f::toVector().
		*/
		void record(const std::vector<float>& headToStage);

		static constexpr float getDegreeWithRadian(const float radian) {
		float radianToDegreeRatio = (180 / PI);
		return (radian * radianToDegreeRatio);
		}

		private:
		static constexpr float PI = M_PI;
		/**
		* Pose recorder time threshold to record sensor data in local log.
		* Sensor data will be recorded into log at least every mPoseRecordThreshold.
		*/
		std::chrono::duration<double> mPoseRecordThreshold;
		// Number of seconds pass since last record.
		std::chrono::duration<double> mNumOfSecondsSinceLastRecord;
		/**
		* According to frameworks/av/media/libheadtracking/include/media/Pose.h
		* "The vector will have exactly 6 elements, where the first three are a translation vector
		* and the last three are a rotation vector."
		*/
		static constexpr size_t mPoseVectorSize = 6;
		/**
		* Timestamp of last sensor data record in local log.
		*/
		std::chrono::time_point<std::chrono::steady_clock> mFirstSampleTimestamp;
		/**
		* Number of sensor samples received since last record, sample rate is ~100Hz which produce
		* ~6k samples/minute.
		*/
		uint32_t mNumOfSampleSinceLastRecord = 0;
		/* The sum of pose angle represented by radian since last dump, div
		* mNumOfSampleSinceLastRecord to get arithmetic mean. Largest possible value: 2PI * 100Hz *
		* mPoseRecordThreshold.
		*/
		std::vector<double> mPoseRadianSum;
		std::vector<float> mMaxPoseAngle;
		std::vector<float> mMinPoseAngle;
		// Local log for history sensor data.
		SimpleLog mPoseRecordLog{mMaxLocalLogLine};

		bool shouldRecordLog() {
		mNumOfSecondsSinceLastRecord = std::chrono::duration_cast<std::chrono::seconds>(
		std::chrono::steady_clock::now() - mFirstSampleTimestamp);
		return mNumOfSecondsSinceLastRecord >= mPoseRecordThreshold;
		}

		void resetRecord() {
		mPoseRadianSum.assign(mPoseVectorSize, 0);
		mMaxPoseAngle.assign(mPoseVectorSize, -PI);
		mMinPoseAngle.assign(mPoseVectorSize, PI);
		mNumOfSampleSinceLastRecord = 0;
		mNumOfSecondsSinceLastRecord = std::chrono::seconds(0);
		}

		// Add each sample to sum and only calculate when record.
		void poseSumToAverage() {
		if (mNumOfSampleSinceLastRecord == 0) return;
		for (auto& p : mPoseRadianSum) {
		const float reciprocal = 1.f / mNumOfSampleSinceLastRecord;
		p *= reciprocal;
		}
		}
		}; // HeadToStagePoseRecorder

		// Record one log line per second (up to mMaxLocalLogLine) to capture most recent sensor data.
		HeadToStagePoseRecorder mPoseRecorder GUARDED_BY(mLock) =
		HeadToStagePoseRecorder(std::chrono::seconds(1), mMaxLocalLogLine);
		media::VectorRecorder mPoseRecorder GUARDED_BY(mLock) {
		6 /* vectorSize */, std::chrono::seconds(1), mMaxLocalLogLine };
		// Record one log line per minute (up to mMaxLocalLogLine) to capture durable sensor data.
		HeadToStagePoseRecorder mPoseDurableRecorder GUARDED_BY(mLock) =
		HeadToStagePoseRecorder(std::chrono::minutes(1), mMaxLocalLogLine);
		media::VectorRecorder mPoseDurableRecorder GUARDED_BY(mLock) {
		6 /* vectorSize */, std::chrono::minutes(1), mMaxLocalLogLine };
		}; // Spatializer

		}; // namespace android