Merge "Add RecordBufferConverter for RecordThread data processing" (77a44e8d) · Commits · e / os / android_frameworks_av

services/audioflinger/RecordTracks.h

+5 −8

Original line number	Diff line number	Diff line
		@@ -34,6 +34,7 @@ public:
		IAudioFlinger::track_flags_t flags,
		track_type type);
		virtual ~RecordTrack();
		virtual status_t initCheck() const;

		virtual status_t start(AudioSystem::sync_event_t event, int triggerSession);
		virtual void stop();
		@@ -66,13 +67,6 @@ private:

		bool mOverflow; // overflow on most recent attempt to fill client buffer

		// updated by RecordThread::readInputParameters_l()
		AudioResampler *mResampler;

		// interleaved stereo pairs of fixed-point Q4.27
		int32_t *mRsmpOutBuffer;
		// current allocated frame count for the above, which may be larger than needed
		size_t mRsmpOutFrameCount;

		size_t mRsmpInUnrel; // unreleased frames remaining from
		// most recent getNextBuffer
		@@ -94,6 +88,9 @@ private:

		// used by resampler to find source frames
		ResamplerBufferProvider *mResamplerBufferProvider;

		// used by the record thread to convert frames to proper destination format
		RecordBufferConverter *mRecordBufferConverter;
		};

		// playback track, used by PatchPanel

services/audioflinger/Threads.cpp

+193 −112

Original line number	Diff line number	Diff line
		@@ -5290,7 +5290,6 @@ failed: ;
		// FIXME mNormalSource
		}


		AudioFlinger::RecordThread::~RecordThread()
		{
		if (mFastCapture != 0) {
		@@ -5594,6 +5593,8 @@ reacquire_wakelock:
		continue;
		}

		// TODO: Update the activeTrack buffer converter in case of reconfigure.

		enum {
		OVERRUN_UNKNOWN,
		OVERRUN_TRUE,
		@@ -5630,109 +5631,9 @@ reacquire_wakelock:
		break;
		}

		if (activeTrack->mResampler == NULL) {
		// no resampling
		if (framesIn > framesOut) {
		framesIn = framesOut;
		} else {
		framesOut = framesIn;
		}
		int8_t *dst = activeTrack->mSink.i8;
		while (framesIn > 0) {
		front &= mRsmpInFramesP2 - 1;
		size_t part1 = mRsmpInFramesP2 - front;
		if (part1 > framesIn) {
		part1 = framesIn;
		}
		int8_t src = (int8_t )mRsmpInBuffer + (front * mFrameSize);
		if (mChannelCount == activeTrack->mChannelCount) {
		memcpy(dst, src, part1 * mFrameSize);
		} else if (mChannelCount == 1) {
		upmix_to_stereo_i16_from_mono_i16((int16_t )dst, (const int16_t )src,
		part1);
		} else {
		downmix_to_mono_i16_from_stereo_i16((int16_t *)dst,
		(const int16_t *)src, part1);
		}
		dst += part1 * activeTrack->mFrameSize;
		front += part1;
		framesIn -= part1;
		}
		activeTrack->mRsmpInFront += framesOut;

		} else {
		// resampling
		// FIXME framesInNeeded should really be part of resampler API, and should
		// depend on the SRC ratio
		// to keep mRsmpInBuffer full so resampler always has sufficient input
		size_t framesInNeeded;
		// FIXME only re-calculate when it changes, and optimize for common ratios
		// Do not precompute in/out because floating point is not associative
		// e.g. ab/c != a(b/c).
		const double in(mSampleRate);
		const double out(activeTrack->mSampleRate);
		framesInNeeded = ceil(framesOut * in / out) + 1;
		ALOGV("need %u frames in to produce %u out given in/out ratio of %.4g",
		framesInNeeded, framesOut, in / out);
		// Although we theoretically have framesIn in circular buffer, some of those are
		// unreleased frames, and thus must be discounted for purpose of budgeting.
		size_t unreleased = activeTrack->mRsmpInUnrel;
		framesIn = framesIn > unreleased ? framesIn - unreleased : 0;
		if (framesIn < framesInNeeded) {
		ALOGV("not enough to resample: have %u frames in but need %u in to "
		"produce %u out given in/out ratio of %.4g",
		framesIn, framesInNeeded, framesOut, in / out);
		size_t newFramesOut = framesIn > 0 ? floor((framesIn - 1) * out / in) : 0;
		LOG_ALWAYS_FATAL_IF(newFramesOut >= framesOut);
		if (newFramesOut == 0) {
		break;
		}
		framesInNeeded = ceil(newFramesOut * in / out) + 1;
		ALOGV("now need %u frames in to produce %u out given out/in ratio of %.4g",
		framesInNeeded, newFramesOut, out / in);
		LOG_ALWAYS_FATAL_IF(framesIn < framesInNeeded);
		ALOGV("success 2: have %u frames in and need %u in to produce %u out "
		"given in/out ratio of %.4g",
		framesIn, framesInNeeded, newFramesOut, in / out);
		framesOut = newFramesOut;
		} else {
		ALOGV("success 1: have %u in and need %u in to produce %u out "
		"given in/out ratio of %.4g",
		framesIn, framesInNeeded, framesOut, in / out);
		}

		// reallocate mRsmpOutBuffer as needed; we will grow but never shrink
		if (activeTrack->mRsmpOutFrameCount < framesOut) {
		// FIXME why does each track need it's own mRsmpOutBuffer? can't they share?
		delete[] activeTrack->mRsmpOutBuffer;
		// resampler always outputs stereo
		activeTrack->mRsmpOutBuffer = new int32_t[framesOut * FCC_2];
		activeTrack->mRsmpOutFrameCount = framesOut;
		}

		// resampler accumulates, but we only have one source track
		memset(activeTrack->mRsmpOutBuffer, 0, framesOut * FCC_2 * sizeof(int32_t));
		activeTrack->mResampler->resample(activeTrack->mRsmpOutBuffer, framesOut,
		// FIXME how about having activeTrack implement this interface itself?
		activeTrack->mResamplerBufferProvider
		/this/ /* AudioBufferProvider* */);
		// ditherAndClamp() works as long as all buffers returned by
		// activeTrack->getNextBuffer() are 32 bit aligned which should be always true.
		if (activeTrack->mChannelCount == 1) {
		// temporarily type pun mRsmpOutBuffer from Q4.27 to int16_t
		ditherAndClamp(activeTrack->mRsmpOutBuffer, activeTrack->mRsmpOutBuffer,
		framesOut);
		// the resampler always outputs stereo samples:
		// do post stereo to mono conversion
		downmix_to_mono_i16_from_stereo_i16(activeTrack->mSink.i16,
		(const int16_t *)activeTrack->mRsmpOutBuffer, framesOut);
		} else {
		ditherAndClamp((int32_t *)activeTrack->mSink.raw,
		activeTrack->mRsmpOutBuffer, framesOut);
		}
		// now done with mRsmpOutBuffer

		}
		// process frames from the RecordThread buffer provider to the RecordTrack buffer
		framesOut = activeTrack->mRecordBufferConverter->convert(
		activeTrack->mSink.raw, activeTrack->mResamplerBufferProvider, framesOut);

		if (framesOut > 0 && (overrun == OVERRUN_UNKNOWN)) {
		overrun = OVERRUN_FALSE;
		@@ -6043,10 +5944,8 @@ status_t AudioFlinger::RecordThread::start(RecordThread::RecordTrack* recordTrac

		recordTrack->mRsmpInFront = mRsmpInRear;
		recordTrack->mRsmpInUnrel = 0;
		// FIXME why reset?
		if (recordTrack->mResampler != NULL) {
		recordTrack->mResampler->reset();
		}
		// clear any converter state as new data will be discontinuous
		recordTrack->mRecordBufferConverter->reset();
		recordTrack->mState = TrackBase::STARTING_2;
		// signal thread to start
		mWaitWorkCV.broadcast();
		@@ -6282,6 +6181,186 @@ void AudioFlinger::RecordThread::ResamplerBufferProvider::releaseBuffer(
		buffer->frameCount = 0;
		}

		AudioFlinger::RecordThread::RecordBufferConverter::RecordBufferConverter(
		audio_channel_mask_t srcChannelMask, audio_format_t srcFormat,
		uint32_t srcSampleRate,
		audio_channel_mask_t dstChannelMask, audio_format_t dstFormat,
		uint32_t dstSampleRate) :
		mSrcChannelMask(AUDIO_CHANNEL_INVALID), // updateParameters will set following vars
		// mSrcFormat
		// mSrcSampleRate
		// mDstChannelMask
		// mDstFormat
		// mDstSampleRate
		// mSrcChannelCount
		// mDstChannelCount
		// mDstFrameSize
		mBuf(NULL), mBufFrames(0), mBufFrameSize(0),
		mResampler(NULL), mRsmpOutBuffer(NULL), mRsmpOutFrameCount(0)
		{
		(void)updateParameters(srcChannelMask, srcFormat, srcSampleRate,
		dstChannelMask, dstFormat, dstSampleRate);
		}

		AudioFlinger::RecordThread::RecordBufferConverter::~RecordBufferConverter() {
		free(mBuf);
		delete mResampler;
		free(mRsmpOutBuffer);
		}

		size_t AudioFlinger::RecordThread::RecordBufferConverter::convert(void *dst,
		AudioBufferProvider *provider, size_t frames)
		{
		if (mSrcSampleRate == mDstSampleRate) {
		ALOGVV("NO RESAMPLING sampleRate:%u mSrcFormat:%#x mDstFormat:%#x",
		mSrcSampleRate, mSrcFormat, mDstFormat);

		AudioBufferProvider::Buffer buffer;
		for (size_t i = frames; i > 0; ) {
		buffer.frameCount = i;
		status_t status = provider->getNextBuffer(&buffer, 0);
		if (status != OK \|\| buffer.frameCount == 0) {
		frames -= i; // cannot fill request.
		break;
		}
		// convert to destination buffer
		convert(dst, buffer.raw, buffer.frameCount);

		dst = (int8_t)dst + buffer.frameCount mDstFrameSize;
		i -= buffer.frameCount;
		provider->releaseBuffer(&buffer);
		}
		} else {
		ALOGVV("RESAMPLING mSrcSampleRate:%u mDstSampleRate:%u mSrcFormat:%#x mDstFormat:%#x",
		mSrcSampleRate, mDstSampleRate, mSrcFormat, mDstFormat);

		// reallocate mRsmpOutBuffer as needed; we will grow but never shrink
		if (mRsmpOutFrameCount < frames) {
		// FIXME why does each track need it's own mRsmpOutBuffer? can't they share?
		free(mRsmpOutBuffer);
		// resampler always outputs stereo (FOR NOW)
		(void)posix_memalign(&mRsmpOutBuffer, 32, frames * FCC_2 * sizeof(int32_t) /Q4.27/);
		mRsmpOutFrameCount = frames;
		}
		// resampler accumulates, but we only have one source track
		memset(mRsmpOutBuffer, 0, frames * FCC_2 * sizeof(int32_t));
		frames = mResampler->resample((int32_t*)mRsmpOutBuffer, frames, provider);

		// convert to destination buffer
		convert(dst, mRsmpOutBuffer, frames);
		}
		return frames;
		}

		status_t AudioFlinger::RecordThread::RecordBufferConverter::updateParameters(
		audio_channel_mask_t srcChannelMask, audio_format_t srcFormat,
		uint32_t srcSampleRate,
		audio_channel_mask_t dstChannelMask, audio_format_t dstFormat,
		uint32_t dstSampleRate)
		{
		// quick evaluation if there is any change.
		if (mSrcFormat == srcFormat
		&& mSrcChannelMask == srcChannelMask
		&& mSrcSampleRate == srcSampleRate
		&& mDstFormat == dstFormat
		&& mDstChannelMask == dstChannelMask
		&& mDstSampleRate == dstSampleRate) {
		return NO_ERROR;
		}

		const bool valid =
		audio_is_input_channel(srcChannelMask)
		&& audio_is_input_channel(dstChannelMask)
		&& audio_is_valid_format(srcFormat) && audio_is_linear_pcm(srcFormat)
		&& audio_is_valid_format(dstFormat) && audio_is_linear_pcm(dstFormat)
		&& (srcSampleRate <= dstSampleRate * AUDIO_RESAMPLER_DOWN_RATIO_MAX)
		; // no upsampling checks for now
		if (!valid) {
		return BAD_VALUE;
		}

		mSrcFormat = srcFormat;
		mSrcChannelMask = srcChannelMask;
		mSrcSampleRate = srcSampleRate;
		mDstFormat = dstFormat;
		mDstChannelMask = dstChannelMask;
		mDstSampleRate = dstSampleRate;

		// compute derived parameters
		mSrcChannelCount = audio_channel_count_from_in_mask(srcChannelMask);
		mDstChannelCount = audio_channel_count_from_in_mask(dstChannelMask);
		mDstFrameSize = mDstChannelCount * audio_bytes_per_sample(mDstFormat);

		// do we need a format buffer?
		if (mSrcFormat != mDstFormat && mDstChannelCount != mSrcChannelCount) {
		mBufFrameSize = mDstChannelCount * audio_bytes_per_sample(mSrcFormat);
		} else {
		mBufFrameSize = 0;
		}
		mBufFrames = 0; // force the buffer to be resized.

		// do we need to resample?
		if (mSrcSampleRate != mDstSampleRate) {
		if (mResampler != NULL) {
		delete mResampler;
		}
		mResampler = AudioResampler::create(AUDIO_FORMAT_PCM_16_BIT,
		mSrcChannelCount, mDstSampleRate); // may seem confusing...
		mResampler->setSampleRate(mSrcSampleRate);
		mResampler->setVolume(AudioMixer::UNITY_GAIN_FLOAT, AudioMixer::UNITY_GAIN_FLOAT);
		}
		return NO_ERROR;
		}

		void AudioFlinger::RecordThread::RecordBufferConverter::convert(
		void dst, /const/ void src, size_t frames)
		{
		// check if a memcpy will do
		if (mResampler == NULL
		&& mSrcChannelCount == mDstChannelCount
		&& mSrcFormat == mDstFormat) {
		memcpy(dst, src,
		frames * mDstChannelCount * audio_bytes_per_sample(mDstFormat));
		return;
		}
		// reallocate buffer if needed
		if (mBufFrameSize != 0 && mBufFrames < frames) {
		free(mBuf);
		mBufFrames = frames;
		(void)posix_memalign(&mBuf, 32, mBufFrames * mBufFrameSize);
		}
		// do processing
		if (mResampler != NULL) {
		// src channel count is always >= 2.
		void *dstBuf = mBuf != NULL ? mBuf : dst;
		// ditherAndClamp() works as long as all buffers returned by
		// activeTrack->getNextBuffer() are 32 bit aligned which should be always true.
		if (mDstChannelCount == 1) {
		// the resampler always outputs stereo samples.
		// FIXME: this rewrites back into src
		ditherAndClamp((int32_t )src, (const int32_t )src, frames);
		downmix_to_mono_i16_from_stereo_i16((int16_t *)dstBuf,
		(const int16_t *)src, frames);
		} else {
		ditherAndClamp((int32_t )dstBuf, (const int32_t )src, frames);
		}
		} else if (mSrcChannelCount != mDstChannelCount) {
		void *dstBuf = mBuf != NULL ? mBuf : dst;
		if (mSrcChannelCount == 1) {
		upmix_to_stereo_i16_from_mono_i16((int16_t )dstBuf, (const int16_t )src,
		frames);
		} else {
		downmix_to_mono_i16_from_stereo_i16((int16_t *)dstBuf,
		(const int16_t *)src, frames);
		}
		}
		if (mSrcFormat != mDstFormat) {
		void *srcBuf = mBuf != NULL ? mBuf : src;
		memcpy_by_audio_format(dst, mDstFormat, srcBuf, mSrcFormat,
		frames * mDstChannelCount);
		}
		}

		bool AudioFlinger::RecordThread::checkForNewParameter_l(const String8& keyValuePair,
		status_t& status)
		{
		@@ -6303,7 +6382,7 @@ bool AudioFlinger::RecordThread::checkForNewParameter_l(const String8& keyValueP
		reconfig = true;
		}
		if (param.getInt(String8(AudioParameter::keyFormat), value) == NO_ERROR) {
		if ((audio_format_t) value != AUDIO_FORMAT_PCM_16_BIT) {
		if (!audio_is_linear_pcm((audio_format_t) value)) {
		status = BAD_VALUE;
		} else {
		reqFormat = (audio_format_t) value;
		@@ -6377,10 +6456,10 @@ bool AudioFlinger::RecordThread::checkForNewParameter_l(const String8& keyValueP
		}
		if (reconfig) {
		if (status == BAD_VALUE &&
		reqFormat == mInput->stream->common.get_format(&mInput->stream->common) &&
		reqFormat == AUDIO_FORMAT_PCM_16_BIT &&
		audio_is_linear_pcm(mInput->stream->common.get_format(&mInput->stream->common)) &&
		audio_is_linear_pcm(reqFormat) &&
		(mInput->stream->common.get_sample_rate(&mInput->stream->common)
		<= (2 * samplingRate)) &&
		<= (AUDIO_RESAMPLER_DOWN_RATIO_MAX * samplingRate)) &&
		audio_channel_count_from_in_mask(
		mInput->stream->common.get_channels(&mInput->stream->common)) <= FCC_2 &&
		(channelMask == AUDIO_CHANNEL_IN_MONO \|\|
		@@ -6451,6 +6530,8 @@ void AudioFlinger::RecordThread::readInputParameters_l()
		// The value is somewhat arbitrary, and could probably be even larger.
		// A larger value should allow more old data to be read after a track calls start(),
		// without increasing latency.
		//
		// Note this is independent of the maximum downsampling ratio permitted for capture.
		mRsmpInFrames = mFrameCount * 7;
		mRsmpInFramesP2 = roundup(mRsmpInFrames);
		delete[] mRsmpInBuffer;

services/audioflinger/Threads.h

+81 −0

Original line number	Diff line number	Diff line
		@@ -1049,6 +1049,87 @@ public:
		RecordTrack * const mRecordTrack;
		};

		/* The RecordBufferConverter is used for format, channel, and sample rate
		* conversion for a RecordTrack.
		*
		* TODO: Self contained, so move to a separate file later.
		*
		* RecordBufferConverter uses the convert() method rather than exposing a
		* buffer provider interface; this is to save a memory copy.
		*/
		class RecordBufferConverter
		{
		public:
		RecordBufferConverter(
		audio_channel_mask_t srcChannelMask, audio_format_t srcFormat,
		uint32_t srcSampleRate,
		audio_channel_mask_t dstChannelMask, audio_format_t dstFormat,
		uint32_t dstSampleRate);

		~RecordBufferConverter();

		/* Converts input data from an AudioBufferProvider by format, channelMask,
		* and sampleRate to a destination buffer.
		*
		* Parameters
		* dst: buffer to place the converted data.
		* provider: buffer provider to obtain source data.
		* frames: number of frames to convert
		*
		* Returns the number of frames converted.
		*/
		size_t convert(void dst, AudioBufferProvider provider, size_t frames);

		// returns NO_ERROR if constructor was successful
		status_t initCheck() const {
		// mSrcChannelMask set on successful updateParameters
		return mSrcChannelMask != AUDIO_CHANNEL_INVALID ? NO_ERROR : NO_INIT;
		}

		// allows dynamic reconfigure of all parameters
		status_t updateParameters(
		audio_channel_mask_t srcChannelMask, audio_format_t srcFormat,
		uint32_t srcSampleRate,
		audio_channel_mask_t dstChannelMask, audio_format_t dstFormat,
		uint32_t dstSampleRate);

		// called to reset resampler buffers on record track discontinuity
		void reset() {
		if (mResampler != NULL) {
		mResampler->reset();
		}
		}

		private:
		// internal convert function for format and channel mask.
		void convert(void dst, /const/ void src, size_t frames);

		// user provided information
		audio_channel_mask_t mSrcChannelMask;
		audio_format_t mSrcFormat;
		uint32_t mSrcSampleRate;
		audio_channel_mask_t mDstChannelMask;
		audio_format_t mDstFormat;
		uint32_t mDstSampleRate;

		// derived information
		uint32_t mSrcChannelCount;
		uint32_t mDstChannelCount;
		size_t mDstFrameSize;

		// format conversion buffer
		void *mBuf;
		size_t mBufFrames;
		size_t mBufFrameSize;

		// resampler info
		AudioResampler *mResampler;
		// interleaved stereo pairs of fixed-point Q4.27 or float depending on resampler
		void *mRsmpOutBuffer;
		// current allocated frame count for the above, which may be larger than needed
		size_t mRsmpOutFrameCount;
		};

		#include "RecordTracks.h"

		RecordThread(const sp<AudioFlinger>& audioFlinger,

services/audioflinger/Tracks.cpp

+26 −16

Original line number	Diff line number	Diff line
		@@ -1990,7 +1990,7 @@ AudioFlinger::RecordThread::RecordTrack::RecordTrack(
		((flags & IAudioFlinger::TRACK_FAST) ? ALLOC_PIPE : ALLOC_CBLK) :
		((buffer == NULL) ? ALLOC_LOCAL : ALLOC_NONE),
		type),
		mOverflow(false), mResampler(NULL), mRsmpOutBuffer(NULL), mRsmpOutFrameCount(0),
		mOverflow(false),
		// See real initialization of mRsmpInFront at RecordThread::start()
		mRsmpInUnrel(0), mRsmpInFront(0), mFramesToDrop(0), mResamplerBufferProvider(NULL)
		{
		@@ -1998,21 +1998,23 @@ AudioFlinger::RecordThread::RecordTrack::RecordTrack(
		return;
		}

		mRecordBufferConverter = new RecordBufferConverter(
		thread->mChannelMask, thread->mFormat, thread->mSampleRate,
		channelMask, format, sampleRate);
		// Check if the RecordBufferConverter construction was successful.
		// If not, don't continue with construction.
		//
		// NOTE: It would be extremely rare that the record track cannot be created
		// for the current device, but a pending or future device change would make
		// the record track configuration valid.
		if (mRecordBufferConverter->initCheck() != NO_ERROR) {
		ALOGE("RecordTrack unable to create record buffer converter");
		return;
		}

		mServerProxy = new AudioRecordServerProxy(mCblk, mBuffer, frameCount,
		mFrameSize, !isExternalTrack());

		uint32_t channelCount = audio_channel_count_from_in_mask(channelMask);
		// FIXME I don't understand either of the channel count checks
		if (thread->mSampleRate != sampleRate && thread->mChannelCount <= FCC_2 &&
		channelCount <= FCC_2) {
		// sink SR
		mResampler = AudioResampler::create(AUDIO_FORMAT_PCM_16_BIT,
		thread->mChannelCount, sampleRate);
		// source SR
		mResampler->setSampleRate(thread->mSampleRate);
		mResampler->setVolume(AudioMixer::UNITY_GAIN_FLOAT, AudioMixer::UNITY_GAIN_FLOAT);
		mResamplerBufferProvider = new ResamplerBufferProvider(this);
		}

		if (flags & IAudioFlinger::TRACK_FAST) {
		ALOG_ASSERT(thread->mFastTrackAvail);
		@@ -2023,11 +2025,19 @@ AudioFlinger::RecordThread::RecordTrack::RecordTrack(
		AudioFlinger::RecordThread::RecordTrack::~RecordTrack()
		{
		ALOGV("%s", __func__);
		delete mResampler;
		delete[] mRsmpOutBuffer;
		delete mRecordBufferConverter;
		delete mResamplerBufferProvider;
		}

		status_t AudioFlinger::RecordThread::RecordTrack::initCheck() const
		{
		status_t status = TrackBase::initCheck();
		if (status == NO_ERROR && mServerProxy == 0) {
		status = BAD_VALUE;
		}
		return status;
		}

		// AudioBufferProvider interface
		status_t AudioFlinger::RecordThread::RecordTrack::getNextBuffer(AudioBufferProvider::Buffer* buffer,
		int64_t pts __unused)