Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 82a4eab1 authored by Andy Hung's avatar Andy Hung
Browse files

Spatial Audio: Generalize VectorRecorder for logging

Move from audiopolicy/service/Spatializer.h.
The VectorRecorder class can be used for head tracking and
sensor recording as well.

Test: adb shell dumpsys media.audio_policy
Bug: 269620212
Merged-In: I7f94932e135fcb5f194ed85b75e4b1234d1d2903
Change-Id: I7f94932e135fcb5f194ed85b75e4b1234d1d2903
parent d2fa7d4c
Loading
Loading
Loading
Loading
+4 −0
Original line number Diff line number Diff line
@@ -21,6 +21,7 @@ cc_library {
      "ScreenHeadFusion.cpp",
      "StillnessDetector.cpp",
      "Twist.cpp",
      "VectorRecorder.cpp",
    ],
    shared_libs: [
        "libaudioutils",
@@ -35,6 +36,9 @@ cc_library {
    export_header_lib_headers: [
        "libeigen",
    ],
    cflags: [
        "-Wthread-safety",
    ],
}

cc_library {
+88 −0
Original line number Diff line number Diff line
/*
 * Copyright (C) 2023 The Android Open Source Project
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

#include "media/VectorRecorder.h"

namespace android::media {

// Convert data to string with level indentation.
// No need for a lock as the SimpleLog is thread-safe.
std::string VectorRecorder::toString(size_t indent) const {
    return mRecordLog.dumpToString(std::string(indent + 1, ' ').c_str(), mMaxLocalLogLine);
}

// Record into local log when it is time.
void VectorRecorder::record(const std::vector<float>& record) {
    if (record.size() != mVectorSize) return;

    // Protect against concurrent calls to record().
    std::lock_guard lg(mLock);

    // if it is time, record average data and reset.
    if (shouldRecordLog_l()) {
        sumToAverage_l();
        mRecordLog.log(
                "mean: %s, min: %s, max %s, calculated %zu samples in %0.4f second(s)",
                toString(mSum).c_str(),
                toString(mMin).c_str(),
                toString(mMax).c_str(),
                mNumberOfSamples,
                mNumberOfSecondsSinceFirstSample.count());
        resetRecord_l();
    }

    // update stream average.
    if (mNumberOfSamples++ == 0) {
        mFirstSampleTimestamp = std::chrono::steady_clock::now();
        for (size_t i = 0; i < mVectorSize; ++i) {
            const float value = record[i];
            mSum[i] += value;
            mMax[i] = value;
            mMin[i] = value;
        }
    } else {
        for (size_t i = 0; i < mVectorSize; ++i) {
            const float value = record[i];
            mSum[i] += value;
            mMax[i] = std::max(mMax[i], value);
            mMin[i] = std::min(mMin[i], value);
        }
    }
}

bool VectorRecorder::shouldRecordLog_l() {
    mNumberOfSecondsSinceFirstSample = std::chrono::duration_cast<std::chrono::seconds>(
            std::chrono::steady_clock::now() - mFirstSampleTimestamp);
    return mNumberOfSecondsSinceFirstSample >= mRecordThreshold;
}

void VectorRecorder::resetRecord_l() {
    mSum.assign(mVectorSize, 0);
    mMax.assign(mVectorSize, 0);
    mMin.assign(mVectorSize, 0);
    mNumberOfSamples = 0;
    mNumberOfSecondsSinceFirstSample = std::chrono::seconds(0);
}

void VectorRecorder::sumToAverage_l() {
    if (mNumberOfSamples == 0) return;
    const float reciprocal = 1.f / mNumberOfSamples;
    for (auto& p : mSum) {
        p *= reciprocal;
    }
}

}  // namespace android::media
+114 −0
Original line number Diff line number Diff line
/*
 * Copyright (C) 2023 The Android Open Source Project
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

#pragma once

#include <android-base/stringprintf.h>
#include <android-base/thread_annotations.h>
#include <audio_utils/SimpleLog.h>
#include <chrono>
#include <math.h>
#include <mutex>
#include <vector>

namespace android::media {

/**
 * VectorRecorder records a vector of floats computing the average, max, and min
 * over given time periods.
 *
 * The class is thread-safe.
 */
class VectorRecorder {
  public:
    VectorRecorder(
        size_t vectorSize, std::chrono::duration<double> threshold, int maxLogLine)
        : mVectorSize(vectorSize)
        , mRecordLog(maxLogLine)
        , mRecordThreshold(threshold)
    {
        resetRecord_l();  // OK to call - we're in the constructor.
    }

    /** Convert recorded vector data to string with level indentation */
    std::string toString(size_t indent) const;

    /**
     * @brief Record a vector of floats.
     *
     * @param record a vector of floats.
     */
    void record(const std::vector<float>& record);

    /**
     * Format vector to a string, [0.00, 0.00, 0.00, -1.29, -0.50, 15.27].
     */
    template <typename T>
    static std::string toString(const std::vector<T>& record) {
        if (record.size() == 0) {
            return "[]";
        }

        std::string ss = "[";
        for (size_t i = 0; i < record.size(); ++i) {
            if (i > 0) {
                ss.append(", ");
            }
            base::StringAppendF(&ss, "%0.2lf", static_cast<double>(record[i]));
        }
        ss.append("]");
        return ss;
    }

  private:
    static constexpr int mMaxLocalLogLine = 10;

    const size_t mVectorSize;

    // Local log for historical vector data.
    // Locked internally, so does not need mutex below.
    SimpleLog mRecordLog{mMaxLocalLogLine};

    std::mutex mLock;

    // Time threshold to record vectors in the local log.
    // Vector data will be recorded into log at least every mRecordThreshold.
    std::chrono::duration<double> mRecordThreshold GUARDED_BY(mLock);

    // Number of seconds since first sample in mSum.
    std::chrono::duration<double> mNumberOfSecondsSinceFirstSample GUARDED_BY(mLock);

    // Timestamp of first sample recorded in mSum.
    std::chrono::time_point<std::chrono::steady_clock> mFirstSampleTimestamp GUARDED_BY(mLock);

    // Number of samples in mSum.
    size_t mNumberOfSamples GUARDED_BY(mLock) = 0;

    std::vector<double> mSum GUARDED_BY(mLock);
    std::vector<float> mMax GUARDED_BY(mLock);
    std::vector<float> mMin GUARDED_BY(mLock);

    // Computes mNumberOfSecondsSinceFirstSample, returns true if time to record.
    bool shouldRecordLog_l() REQUIRES(mLock);

    // Resets the running mNumberOfSamples, mSum, mMax, mMin.
    void resetRecord_l() REQUIRES(mLock);

    // Convert mSum to an average.
    void sumToAverage_l() REQUIRES(mLock);
};  // VectorRecorder

}  // namespace android::media
+12 −36
Original line number Diff line number Diff line
@@ -75,6 +75,16 @@ static audio_channel_mask_t getMaxChannelMask(
    return maxMask;
}

std::vector<float> recordFromRotationVector(const std::vector<float>& rotationVector) {
    constexpr float RAD_TO_DEGREE = 180.f / M_PI;
    std::vector<float> record{
        rotationVector[0], rotationVector[1], rotationVector[2],
        rotationVector[3] * RAD_TO_DEGREE,
        rotationVector[4] * RAD_TO_DEGREE,
        rotationVector[5] * RAD_TO_DEGREE};
    return record;
}

// ---------------------------------------------------------------------------

class Spatializer::EngineCallbackHandler : public AHandler {
@@ -184,41 +194,6 @@ const std::vector<const char *> Spatializer::sHeadPoseKeys = {
    Spatializer::EngineCallbackHandler::kRotation2Key,
};

// ---------------------------------------------------------------------------

// Convert recorded sensor data to string with level indentation.
std::string Spatializer::HeadToStagePoseRecorder::toString(unsigned level) const {
    std::string prefixSpace(level, ' ');
    return mPoseRecordLog.dumpToString((prefixSpace + " ").c_str(), Spatializer::mMaxLocalLogLine);
}

// Compute sensor data, record into local log when it is time.
void Spatializer::HeadToStagePoseRecorder::record(const std::vector<float>& headToStage) {
    if (headToStage.size() != mPoseVectorSize) return;

    if (mNumOfSampleSinceLastRecord++ == 0) {
        mFirstSampleTimestamp = std::chrono::steady_clock::now();
    }
    // if it's time, do record and reset.
    if (shouldRecordLog()) {
        poseSumToAverage();
        mPoseRecordLog.log(
                "mean: %s, min: %s, max %s, calculated %d samples in %0.4f second(s)",
                Spatializer::toString<double>(mPoseRadianSum, true /* radianToDegree */).c_str(),
                Spatializer::toString<float>(mMinPoseAngle, true /* radianToDegree */).c_str(),
                Spatializer::toString<float>(mMaxPoseAngle, true /* radianToDegree */).c_str(),
                mNumOfSampleSinceLastRecord, mNumOfSecondsSinceLastRecord.count());
        resetRecord();
    }
    // update stream average.
    for (int i = 0; i < mPoseVectorSize; i++) {
        mPoseRadianSum[i] += headToStage[i];
        mMaxPoseAngle[i] = std::max(mMaxPoseAngle[i], headToStage[i]);
        mMinPoseAngle[i] = std::min(mMinPoseAngle[i], headToStage[i]);
    }
    return;
}

// ---------------------------------------------------------------------------
sp<Spatializer> Spatializer::create(SpatializerPolicyCallback *callback) {
    sp<Spatializer> spatializer;
@@ -590,7 +565,8 @@ Status Spatializer::setGlobalTransform(const std::vector<float>& screenToStage)
    }
    std::lock_guard lock(mLock);
    if (mPoseController != nullptr) {
        mLocalLog.log("%s with screenToStage %s", __func__, toString<float>(screenToStage).c_str());
        mLocalLog.log("%s with screenToStage %s", __func__,
                media::VectorRecorder::toString<float>(screenToStage).c_str());
        mPoseController->setScreenToStagePose(maybePose.value());
    }
    return Status::ok();
+5 −108
Original line number Diff line number Diff line
@@ -27,6 +27,7 @@
#include <audio_utils/SimpleLog.h>
#include <math.h>
#include <media/AudioEffect.h>
#include <media/VectorRecorder.h>
#include <media/audiohal/EffectHalInterface.h>
#include <media/stagefright/foundation/ALooper.h>
#include <system/audio_effects/effect_spatializer.h>
@@ -172,30 +173,6 @@ class Spatializer : public media::BnSpatializer,
                media::audio::common::toString(*result) : "unknown_latency_mode";
    }

    /**
     * Format head to stage vector to a string, [0.00, 0.00, 0.00, -1.29, -0.50, 15.27].
     */
    template <typename T>
    static std::string toString(const std::vector<T>& vec, bool radianToDegree = false) {
        if (vec.size() == 0) {
            return "[]";
        }

        std::string ss = "[";
        for (auto f = vec.begin(); f != vec.end(); ++f) {
            if (f != vec.begin()) {
                ss .append(", ");
            }
            if (radianToDegree) {
                base::StringAppendF(&ss, "%0.2f", HeadToStagePoseRecorder::getDegreeWithRadian(*f));
            } else {
                base::StringAppendF(&ss, "%f", *f);
            }
        }
        ss.append("]");
        return ss;
    };

    // If the Spatializer is not created, we send the status for metrics purposes.
    // OK:      Spatializer not expected to be created.
    // NO_INIT: Spatializer creation failed.
@@ -427,92 +404,12 @@ private:
     * @brief Calculate and record sensor data.
     * Dump to local log with max/average pose angle every mPoseRecordThreshold.
     */
    class HeadToStagePoseRecorder {
      public:
        HeadToStagePoseRecorder(std::chrono::duration<double> threshold, int maxLogLine)
            : mPoseRecordThreshold(threshold), mPoseRecordLog(maxLogLine) {
            resetRecord();
        }

        /** Convert recorded sensor data to string with level indentation */
        std::string toString(unsigned level) const;

        /**
         * @brief Calculate sensor data, record into local log when it is time.
         *
         * @param headToStage The vector from Pose3f::toVector().
         */
        void record(const std::vector<float>& headToStage);

        static constexpr float getDegreeWithRadian(const float radian) {
            float radianToDegreeRatio = (180 / PI);
            return (radian * radianToDegreeRatio);
        }

      private:
        static constexpr float PI = M_PI;
        /**
         * Pose recorder time threshold to record sensor data in local log.
         * Sensor data will be recorded into log at least every mPoseRecordThreshold.
         */
        std::chrono::duration<double> mPoseRecordThreshold;
        // Number of seconds pass since last record.
        std::chrono::duration<double> mNumOfSecondsSinceLastRecord;
        /**
         * According to frameworks/av/media/libheadtracking/include/media/Pose.h
         * "The vector will have exactly 6 elements, where the first three are a translation vector
         * and the last three are a rotation vector."
         */
        static constexpr size_t mPoseVectorSize = 6;
        /**
         * Timestamp of last sensor data record in local log.
         */
        std::chrono::time_point<std::chrono::steady_clock> mFirstSampleTimestamp;
        /**
         * Number of sensor samples received since last record, sample rate is ~100Hz which produce
         * ~6k samples/minute.
         */
        uint32_t mNumOfSampleSinceLastRecord = 0;
        /* The sum of pose angle represented by radian since last dump, div
         * mNumOfSampleSinceLastRecord to get arithmetic mean. Largest possible value: 2PI * 100Hz *
         * mPoseRecordThreshold.
         */
        std::vector<double> mPoseRadianSum;
        std::vector<float> mMaxPoseAngle;
        std::vector<float> mMinPoseAngle;
        // Local log for history sensor data.
        SimpleLog mPoseRecordLog{mMaxLocalLogLine};

        bool shouldRecordLog() {
            mNumOfSecondsSinceLastRecord = std::chrono::duration_cast<std::chrono::seconds>(
                    std::chrono::steady_clock::now() - mFirstSampleTimestamp);
            return mNumOfSecondsSinceLastRecord >= mPoseRecordThreshold;
        }

        void resetRecord() {
            mPoseRadianSum.assign(mPoseVectorSize, 0);
            mMaxPoseAngle.assign(mPoseVectorSize, -PI);
            mMinPoseAngle.assign(mPoseVectorSize, PI);
            mNumOfSampleSinceLastRecord = 0;
            mNumOfSecondsSinceLastRecord = std::chrono::seconds(0);
        }

        // Add each sample to sum and only calculate when record.
        void poseSumToAverage() {
            if (mNumOfSampleSinceLastRecord == 0) return;
            for (auto& p : mPoseRadianSum) {
                const float reciprocal = 1.f / mNumOfSampleSinceLastRecord;
                p *= reciprocal;
            }
        }
    };  // HeadToStagePoseRecorder

    // Record one log line per second (up to mMaxLocalLogLine) to capture most recent sensor data.
    HeadToStagePoseRecorder mPoseRecorder GUARDED_BY(mLock) =
            HeadToStagePoseRecorder(std::chrono::seconds(1), mMaxLocalLogLine);
    media::VectorRecorder mPoseRecorder GUARDED_BY(mLock) {
        6 /* vectorSize */, std::chrono::seconds(1), mMaxLocalLogLine };
    // Record one log line per minute (up to mMaxLocalLogLine) to capture durable sensor data.
    HeadToStagePoseRecorder mPoseDurableRecorder GUARDED_BY(mLock) =
            HeadToStagePoseRecorder(std::chrono::minutes(1), mMaxLocalLogLine);
    media::VectorRecorder mPoseDurableRecorder  GUARDED_BY(mLock) {
        6 /* vectorSize */, std::chrono::minutes(1), mMaxLocalLogLine };
};  // Spatializer

}; // namespace android