Merge "Transcoder: Added MediaTrackTranscoder and VideoTrackTranscoder" (da34ac29) · Commits · e / os / android_frameworks_av

media/libmediatranscoding/transcoder/Android.bp

+3 −0

Original line number	Diff line number	Diff line
		@@ -20,12 +20,15 @@ cc_library_shared {
		srcs: [
		"MediaSampleQueue.cpp",
		"MediaSampleReaderNDK.cpp",
		"MediaTrackTranscoder.cpp",
		"VideoTrackTranscoder.cpp",
		],

		shared_libs: [
		"libbase",
		"libcutils",
		"libmediandk",
		"libnativewindow",
		"libutils",
		],

media/libmediatranscoding/transcoder/MediaSampleReaderNDK.cpp

+4 −0

Original line number	Diff line number	Diff line
		@@ -25,6 +25,10 @@

		namespace android {

		// Check that the extractor sample flags have the expected NDK meaning.
		static_assert(SAMPLE_FLAG_SYNC_SAMPLE == AMEDIAEXTRACTOR_SAMPLE_FLAG_SYNC,
		"Sample flag mismatch: SYNC_SAMPLE");

		// static
		std::shared_ptr<MediaSampleReader> MediaSampleReaderNDK::createFromFd(int fd, size_t offset,
		size_t size) {

media/libmediatranscoding/transcoder/MediaTrackTranscoder.cpp

0 → 100644

+104 −0

Original line number	Diff line number	Diff line
		/*
		* Copyright (C) 2020 The Android Open Source Project
		*
		* Licensed under the Apache License, Version 2.0 (the "License");
		* you may not use this file except in compliance with the License.
		* You may obtain a copy of the License at
		*
		* http://www.apache.org/licenses/LICENSE-2.0
		*
		* Unless required by applicable law or agreed to in writing, software
		* distributed under the License is distributed on an "AS IS" BASIS,
		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
		* See the License for the specific language governing permissions and
		* limitations under the License.
		*/

		// #define LOG_NDEBUG 0
		#define LOG_TAG "MediaTrackTranscoder"

		#include <android-base/logging.h>
		#include <media/MediaTrackTranscoder.h>

		namespace android {

		media_status_t MediaTrackTranscoder::configure(
		const std::shared_ptr<MediaSampleReader>& mediaSampleReader, int trackIndex,
		const std::shared_ptr<AMediaFormat>& destinationFormat) {
		std::scoped_lock lock{mStateMutex};

		if (mState != UNINITIALIZED) {
		LOG(ERROR) << "Configure can only be called once";
		return AMEDIA_ERROR_UNSUPPORTED;
		}

		if (mediaSampleReader == nullptr) {
		LOG(ERROR) << "MediaSampleReader is null";
		return AMEDIA_ERROR_INVALID_PARAMETER;
		}
		if (trackIndex < 0 \|\| trackIndex >= mediaSampleReader->getTrackCount()) {
		LOG(ERROR) << "TrackIndex is invalid " << trackIndex;
		return AMEDIA_ERROR_INVALID_PARAMETER;
		}

		mMediaSampleReader = mediaSampleReader;
		mTrackIndex = trackIndex;

		mSourceFormat =
		std::shared_ptr<AMediaFormat>(mMediaSampleReader->getTrackFormat(mTrackIndex),
		std::bind(AMediaFormat_delete, std::placeholders::_1));
		if (mSourceFormat == nullptr) {
		LOG(ERROR) << "Unable to get format for track #" << mTrackIndex;
		return AMEDIA_ERROR_MALFORMED;
		}

		media_status_t status = configureDestinationFormat(destinationFormat);
		if (status != AMEDIA_OK) {
		LOG(ERROR) << "configure failed with error " << status;
		return status;
		}

		mState = CONFIGURED;
		return AMEDIA_OK;
		}

		bool MediaTrackTranscoder::start() {
		std::scoped_lock lock{mStateMutex};

		if (mState != CONFIGURED) {
		LOG(ERROR) << "TrackTranscoder must be configured before started";
		return false;
		}

		mTranscodingThread = std::thread([this] {
		media_status_t status = runTranscodeLoop();

		// Notify the client.
		if (auto callbacks = mTranscoderCallback.lock()) {
		if (status != AMEDIA_OK) {
		callbacks->onTrackError(this, status);
		} else {
		callbacks->onTrackFinished(this);
		}
		}
		});

		mState = STARTED;
		return true;
		}

		bool MediaTrackTranscoder::stop() {
		std::scoped_lock lock{mStateMutex};

		if (mState == STARTED) {
		abortTranscodeLoop();
		mTranscodingThread.join();
		mState = STOPPED;
		return true;
		}

		LOG(ERROR) << "TrackTranscoder must be started before stopped";
		return false;
		}

		} // namespace android
		No newline at end of file

media/libmediatranscoding/transcoder/VideoTrackTranscoder.cpp

0 → 100644

+335 −0

Original line number	Diff line number	Diff line
		/*
		* Copyright (C) 2020 The Android Open Source Project
		*
		* Licensed under the Apache License, Version 2.0 (the "License");
		* you may not use this file except in compliance with the License.
		* You may obtain a copy of the License at
		*
		* http://www.apache.org/licenses/LICENSE-2.0
		*
		* Unless required by applicable law or agreed to in writing, software
		* distributed under the License is distributed on an "AS IS" BASIS,
		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
		* See the License for the specific language governing permissions and
		* limitations under the License.
		*/

		// #define LOG_NDEBUG 0
		#define LOG_TAG "VideoTrackTranscoder"

		#include <android-base/logging.h>
		#include <media/VideoTrackTranscoder.h>

		namespace android {

		// Check that the codec sample flags have the expected NDK meaning.
		static_assert(SAMPLE_FLAG_CODEC_CONFIG == AMEDIACODEC_BUFFER_FLAG_CODEC_CONFIG,
		"Sample flag mismatch: CODEC_CONFIG");
		static_assert(SAMPLE_FLAG_END_OF_STREAM == AMEDIACODEC_BUFFER_FLAG_END_OF_STREAM,
		"Sample flag mismatch: END_OF_STREAM");
		static_assert(SAMPLE_FLAG_PARTIAL_FRAME == AMEDIACODEC_BUFFER_FLAG_PARTIAL_FRAME,
		"Sample flag mismatch: PARTIAL_FRAME");

		template <typename T>
		void VideoTrackTranscoder::BlockingQueue<T>::push(T const& value, bool front) {
		{
		std::unique_lock<std::mutex> lock(mMutex);
		if (front) {
		mQueue.push_front(value);
		} else {
		mQueue.push_back(value);
		}
		}
		mCondition.notify_one();
		}

		template <typename T>
		T VideoTrackTranscoder::BlockingQueue<T>::pop() {
		std::unique_lock<std::mutex> lock(mMutex);
		while (mQueue.empty()) {
		mCondition.wait(lock);
		}
		T value = mQueue.front();
		mQueue.pop_front();
		return value;
		}

		// Dispatch responses to codec callbacks onto the message queue.
		struct AsyncCodecCallbackDispatch {
		static void onAsyncInputAvailable(AMediaCodec* codec, void* userdata, int32_t index) {
		VideoTrackTranscoder* transcoder = static_cast<VideoTrackTranscoder*>(userdata);
		if (codec == transcoder->mDecoder) {
		transcoder->mCodecMessageQueue.push(
		[transcoder, index] { transcoder->enqueueInputSample(index); });
		}
		}

		static void onAsyncOutputAvailable(AMediaCodec* codec, void* userdata, int32_t index,
		AMediaCodecBufferInfo* bufferInfoPtr) {
		VideoTrackTranscoder* transcoder = static_cast<VideoTrackTranscoder*>(userdata);
		AMediaCodecBufferInfo bufferInfo = *bufferInfoPtr;
		transcoder->mCodecMessageQueue.push([transcoder, index, codec, bufferInfo] {
		if (codec == transcoder->mDecoder) {
		transcoder->transferBuffer(index, bufferInfo);
		} else if (codec == transcoder->mEncoder) {
		transcoder->dequeueOutputSample(index, bufferInfo);
		}
		});
		}

		static void onAsyncFormatChanged(AMediaCodec* codec, void* userdata, AMediaFormat* format) {
		VideoTrackTranscoder* transcoder = static_cast<VideoTrackTranscoder*>(userdata);
		const char* kCodecName = (codec == transcoder->mDecoder ? "Decoder" : "Encoder");
		LOG(DEBUG) << kCodecName << " format changed: " << AMediaFormat_toString(format);
		}

		static void onAsyncError(AMediaCodec* codec, void* userdata, media_status_t error,
		int32_t actionCode, const char* detail) {
		LOG(ERROR) << "Error from codec " << codec << ", userdata " << userdata << ", error "
		<< error << ", action " << actionCode << ", detail " << detail;
		VideoTrackTranscoder* transcoder = static_cast<VideoTrackTranscoder*>(userdata);
		transcoder->mCodecMessageQueue.push(
		[transcoder, error] {
		transcoder->mStatus = error;
		transcoder->mStopRequested = true;
		},
		true);
		}
		};

		VideoTrackTranscoder::~VideoTrackTranscoder() {
		if (mDecoder != nullptr) {
		AMediaCodec_delete(mDecoder);
		}

		if (mEncoder != nullptr) {
		AMediaCodec_delete(mEncoder);
		}

		if (mSurface != nullptr) {
		ANativeWindow_release(mSurface);
		}
		}

		// Creates and configures the codecs.
		media_status_t VideoTrackTranscoder::configureDestinationFormat(
		const std::shared_ptr<AMediaFormat>& destinationFormat) {
		media_status_t status = AMEDIA_OK;

		if (destinationFormat == nullptr) {
		LOG(ERROR) << "Destination format is null";
		return AMEDIA_ERROR_INVALID_PARAMETER;
		}

		mDestinationFormat = destinationFormat;

		// Create and configure the encoder.
		const char* destinationMime = nullptr;
		bool ok = AMediaFormat_getString(mDestinationFormat.get(), AMEDIAFORMAT_KEY_MIME,
		&destinationMime);
		if (!ok) {
		LOG(ERROR) << "Destination MIME type is required for transcoding.";
		return AMEDIA_ERROR_INVALID_PARAMETER;
		}

		mEncoder = AMediaCodec_createEncoderByType(destinationMime);
		if (mEncoder == nullptr) {
		LOG(ERROR) << "Unable to create encoder for type " << destinationMime;
		return AMEDIA_ERROR_UNSUPPORTED;
		}

		status = AMediaCodec_configure(mEncoder, mDestinationFormat.get(), NULL /* surface */,
		NULL /* crypto */, AMEDIACODEC_CONFIGURE_FLAG_ENCODE);
		if (status != AMEDIA_OK) {
		LOG(ERROR) << "Unable to configure video encoder: " << status;
		return status;
		}

		status = AMediaCodec_createInputSurface(mEncoder, &mSurface);
		if (status != AMEDIA_OK) {
		LOG(ERROR) << "Unable to create an encoder input surface: %d" << status;
		return status;
		}

		// Create and configure the decoder.
		const char* sourceMime = nullptr;
		ok = AMediaFormat_getString(mSourceFormat.get(), AMEDIAFORMAT_KEY_MIME, &sourceMime);
		if (!ok) {
		LOG(ERROR) << "Source MIME type is required for transcoding.";
		return AMEDIA_ERROR_INVALID_PARAMETER;
		}

		mDecoder = AMediaCodec_createDecoderByType(sourceMime);
		if (mDecoder == nullptr) {
		LOG(ERROR) << "Unable to create decoder for type " << sourceMime;
		return AMEDIA_ERROR_UNSUPPORTED;
		}

		status = AMediaCodec_configure(mDecoder, mSourceFormat.get(), mSurface, NULL /* crypto */,
		0 /* flags */);
		if (status != AMEDIA_OK) {
		LOG(ERROR) << "Unable to configure video decoder: " << status;
		return status;
		}

		// Configure codecs to run in async mode.
		AMediaCodecOnAsyncNotifyCallback asyncCodecCallbacks = {
		.onAsyncInputAvailable = AsyncCodecCallbackDispatch::onAsyncInputAvailable,
		.onAsyncOutputAvailable = AsyncCodecCallbackDispatch::onAsyncOutputAvailable,
		.onAsyncFormatChanged = AsyncCodecCallbackDispatch::onAsyncFormatChanged,
		.onAsyncError = AsyncCodecCallbackDispatch::onAsyncError};

		status = AMediaCodec_setAsyncNotifyCallback(mDecoder, asyncCodecCallbacks, this);
		if (status != AMEDIA_OK) {
		LOG(ERROR) << "Unable to set decoder to async mode: " << status;
		return status;
		}

		status = AMediaCodec_setAsyncNotifyCallback(mEncoder, asyncCodecCallbacks, this);
		if (status != AMEDIA_OK) {
		LOG(ERROR) << "Unable to set encoder to async mode: " << status;
		return status;
		}

		return AMEDIA_OK;
		}

		void VideoTrackTranscoder::enqueueInputSample(int32_t bufferIndex) {
		media_status_t status = AMEDIA_OK;

		if (mEOSFromSource) {
		return;
		}

		status = mMediaSampleReader->getSampleInfoForTrack(mTrackIndex, &mSampleInfo);
		if (status != AMEDIA_OK && status != AMEDIA_ERROR_END_OF_STREAM) {
		LOG(ERROR) << "Error getting next sample info: " << status;
		mStatus = status;
		return;
		}
		const bool endOfStream = (status == AMEDIA_ERROR_END_OF_STREAM);

		if (!endOfStream) {
		size_t bufferSize = 0;
		uint8_t* sourceBuffer = AMediaCodec_getInputBuffer(mDecoder, bufferIndex, &bufferSize);
		if (sourceBuffer == nullptr) {
		LOG(ERROR) << "Decoder returned a NULL input buffer.";
		mStatus = AMEDIA_ERROR_UNKNOWN;
		return;
		} else if (bufferSize < mSampleInfo.size) {
		LOG(ERROR) << "Decoder returned an input buffer that is smaller than the sample.";
		mStatus = AMEDIA_ERROR_UNKNOWN;
		return;
		}

		status = mMediaSampleReader->readSampleDataForTrack(mTrackIndex, sourceBuffer,
		mSampleInfo.size);
		if (status != AMEDIA_OK) {
		LOG(ERROR) << "Unable to read next sample data. Aborting transcode.";
		mStatus = status;
		return;
		}

		mMediaSampleReader->advanceTrack(mTrackIndex);
		} else {
		LOG(DEBUG) << "EOS from source.";
		mEOSFromSource = true;
		}

		status = AMediaCodec_queueInputBuffer(mDecoder, bufferIndex, 0, mSampleInfo.size,
		mSampleInfo.presentationTimeUs, mSampleInfo.flags);
		if (status != AMEDIA_OK) {
		LOG(ERROR) << "Unable to queue input buffer for decode: " << status;
		mStatus = status;
		return;
		}
		}

		void VideoTrackTranscoder::transferBuffer(int32_t bufferIndex, AMediaCodecBufferInfo bufferInfo) {
		if (bufferIndex >= 0) {
		bool needsRender = bufferInfo.size > 0;
		AMediaCodec_releaseOutputBuffer(mDecoder, bufferIndex, needsRender);
		}

		if (bufferInfo.flags & AMEDIACODEC_BUFFER_FLAG_END_OF_STREAM) {
		LOG(DEBUG) << "EOS from decoder.";
		media_status_t status = AMediaCodec_signalEndOfInputStream(mEncoder);
		if (status != AMEDIA_OK) {
		LOG(ERROR) << "SignalEOS on encoder returned error: " << status;
		mStatus = status;
		}
		}
		}

		void VideoTrackTranscoder::dequeueOutputSample(int32_t bufferIndex,
		AMediaCodecBufferInfo bufferInfo) {
		if (bufferIndex >= 0) {
		size_t sampleSize = 0;
		uint8_t* buffer = AMediaCodec_getOutputBuffer(mEncoder, bufferIndex, &sampleSize);

		std::shared_ptr<MediaSample> sample = MediaSample::createWithReleaseCallback(
		buffer, bufferInfo.offset, bufferIndex,
		std::bind(&VideoTrackTranscoder::releaseOutputSample, this, std::placeholders::_1));
		sample->info.size = bufferInfo.size;
		sample->info.flags = bufferInfo.flags;
		sample->info.presentationTimeUs = bufferInfo.presentationTimeUs;

		const bool aborted = mOutputQueue.enqueue(sample);
		if (aborted) {
		LOG(ERROR) << "Output sample queue was aborted. Stopping transcode.";
		mStatus = AMEDIA_ERROR_IO; // TODO: Define custom error codes?
		return;
		}
		} else if (bufferIndex == AMEDIACODEC_INFO_OUTPUT_FORMAT_CHANGED) {
		AMediaFormat* newFormat = AMediaCodec_getOutputFormat(mEncoder);
		LOG(DEBUG) << "Encoder output format changed: " << AMediaFormat_toString(newFormat);
		}

		if (bufferInfo.flags & AMEDIACODEC_BUFFER_FLAG_END_OF_STREAM) {
		LOG(DEBUG) << "EOS from encoder.";
		mEOSFromEncoder = true;
		}
		}

		void VideoTrackTranscoder::releaseOutputSample(MediaSample* sample) {
		AMediaCodec_releaseOutputBuffer(mEncoder, sample->bufferId, false /* render */);
		}

		media_status_t VideoTrackTranscoder::runTranscodeLoop() {
		media_status_t status = AMEDIA_OK;

		status = AMediaCodec_start(mDecoder);
		if (status != AMEDIA_OK) {
		LOG(ERROR) << "Unable to start video decoder: " << status;
		return status;
		}

		status = AMediaCodec_start(mEncoder);
		if (status != AMEDIA_OK) {
		LOG(ERROR) << "Unable to start video encoder: " << status;
		AMediaCodec_stop(mDecoder);
		return status;
		}

		// Process codec events until EOS is reached, transcoding is stopped or an error occurs.
		while (!mStopRequested && !mEOSFromEncoder && mStatus == AMEDIA_OK) {
		std::function<void()> message = mCodecMessageQueue.pop();
		message();
		}

		// Return error if transcoding was stopped before it finished.
		if (mStopRequested && !mEOSFromEncoder && mStatus == AMEDIA_OK) {
		mStatus = AMEDIA_ERROR_UNKNOWN; // TODO: Define custom error codes?
		}

		AMediaCodec_stop(mDecoder);
		AMediaCodec_stop(mEncoder);
		return mStatus;
		}

		void VideoTrackTranscoder::abortTranscodeLoop() {
		// Push abort message to the front of the codec event queue.
		mCodecMessageQueue.push([this] { mStopRequested = true; }, true /* front */);
		}

		} // namespace android

media/libmediatranscoding/transcoder/include/media/MediaSample.h

+0 −15

Original line number	Diff line number	Diff line
		@@ -38,21 +38,6 @@ enum : uint32_t {
		SAMPLE_FLAG_PARTIAL_FRAME = 8,
		};

		// Check that the sample flags have the expected NDK meaning.
		namespace {
		#include <media/NdkMediaCodec.h>
		#include <media/NdkMediaExtractor.h>

		static_assert(SAMPLE_FLAG_SYNC_SAMPLE == AMEDIAEXTRACTOR_SAMPLE_FLAG_SYNC,
		"Sample flag mismatch: SYNC_SAMPLE");
		static_assert(SAMPLE_FLAG_CODEC_CONFIG == AMEDIACODEC_BUFFER_FLAG_CODEC_CONFIG,
		"Sample flag mismatch: CODEC_CONFIG");
		static_assert(SAMPLE_FLAG_END_OF_STREAM == AMEDIACODEC_BUFFER_FLAG_END_OF_STREAM,
		"Sample flag mismatch: END_OF_STREAM");
		static_assert(SAMPLE_FLAG_PARTIAL_FRAME == AMEDIACODEC_BUFFER_FLAG_PARTIAL_FRAME,
		"Sample flag mismatch: PARTIAL_FRAME");
		} // anonymous namespace

		/**
		* MediaSampleInfo is an object that carries information about a compressed media sample without
		* holding any sample data.