Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit a676a9a1 authored by Linus Nilsson's avatar Linus Nilsson
Browse files

Transcoder: Don't preserve profile/level when switching codec. 

Avoid configuring encoder with source file's profile and level
if the codec type itself will change. Instead leave the default
unspecified. The caller can still set profile/level when
configuring the transcoder.

Bug: 182576621
Test: Transcoder unit tests.
Change-Id: I469a7a397ebf3e2eade6a48d95851656157578b7
parent 52df71c2

media/libmediatranscoding/transcoder/MediaTranscoder.cpp

+42 −29
Original line number Diff line number Diff line
@@ -29,24 +29,37 @@ 


namespace android {



static AMediaFormat* mergeMediaFormats(AMediaFormat* base, AMediaFormat* overlay) {

    if (base == nullptr || overlay == nullptr) {

static std::shared_ptr<AMediaFormat> createVideoTrackFormat(AMediaFormat* srcFormat,

                                                            AMediaFormat* options) {

    if (srcFormat == nullptr || options == nullptr) {

        LOG(ERROR) << "Cannot merge null formats";

        return nullptr;

    }



    AMediaFormat* format = AMediaFormat_new();

    if (AMediaFormat_copy(format, base) != AMEDIA_OK) {

        AMediaFormat_delete(format);

        return nullptr;

    // ------- Define parameters to copy from the source track format -------

    std::vector<AMediaFormatUtils::EntryCopier> srcParamsToCopy{

            ENTRY_COPIER(AMEDIAFORMAT_KEY_MIME, String),

            ENTRY_COPIER(AMEDIAFORMAT_KEY_DURATION, Int64),

            ENTRY_COPIER(AMEDIAFORMAT_KEY_WIDTH, Int32),

            ENTRY_COPIER(AMEDIAFORMAT_KEY_HEIGHT, Int32),

            ENTRY_COPIER(AMEDIAFORMAT_KEY_FRAME_RATE, Int32),

            ENTRY_COPIER(AMEDIAFORMAT_KEY_COLOR_RANGE, Int32),

            ENTRY_COPIER(AMEDIAFORMAT_KEY_COLOR_STANDARD, Int32),

            ENTRY_COPIER(AMEDIAFORMAT_KEY_COLOR_TRANSFER, Int32),

    };



    // If the destination codec is the same as the source codec, we can preserve profile and level

    // from the source track as default values. Otherwise leave them unspecified.

    const char *srcMime, *dstMime;

    AMediaFormat_getString(srcFormat, AMEDIAFORMAT_KEY_MIME, &srcMime);

    if (!AMediaFormat_getString(options, AMEDIAFORMAT_KEY_MIME, &dstMime) ||

        strcmp(srcMime, dstMime) == 0) {

        srcParamsToCopy.push_back(ENTRY_COPIER(AMEDIAFORMAT_KEY_PROFILE, String));

        srcParamsToCopy.push_back(ENTRY_COPIER(AMEDIAFORMAT_KEY_LEVEL, String));

    }



    // Note: AMediaFormat does not expose a function for appending values from another format or for

    // iterating over all values and keys in a format. Instead we define a static list of known keys

    // along with their value types and copy the ones that are present. A better solution would be

    // to either implement required functions in NDK or to parse the overlay format's string

    // representation and copy all existing keys.

    static const AMediaFormatUtils::EntryCopier kSupportedFormatEntries[] = {

    // ------- Define parameters to copy from the caller's options -------

    static const std::vector<AMediaFormatUtils::EntryCopier> kSupportedOptions{

            ENTRY_COPIER(AMEDIAFORMAT_KEY_MIME, String),

            ENTRY_COPIER(AMEDIAFORMAT_KEY_DURATION, Int64),

            ENTRY_COPIER(AMEDIAFORMAT_KEY_WIDTH, Int32),
@@ -54,7 +67,6 @@ static AMediaFormat* mergeMediaFormats(AMediaFormat* base, AMediaFormat* overlay 
            ENTRY_COPIER(AMEDIAFORMAT_KEY_BIT_RATE, Int32),

            ENTRY_COPIER(AMEDIAFORMAT_KEY_PROFILE, Int32),

            ENTRY_COPIER(AMEDIAFORMAT_KEY_LEVEL, Int32),

            ENTRY_COPIER(AMEDIAFORMAT_KEY_COLOR_FORMAT, Int32),

            ENTRY_COPIER(AMEDIAFORMAT_KEY_COLOR_RANGE, Int32),

            ENTRY_COPIER(AMEDIAFORMAT_KEY_COLOR_STANDARD, Int32),

            ENTRY_COPIER(AMEDIAFORMAT_KEY_COLOR_TRANSFER, Int32),
@@ -63,10 +75,12 @@ static AMediaFormat* mergeMediaFormats(AMediaFormat* base, AMediaFormat* overlay 
            ENTRY_COPIER(AMEDIAFORMAT_KEY_PRIORITY, Int32),

            ENTRY_COPIER2(AMEDIAFORMAT_KEY_OPERATING_RATE, Float, Int32),

    };

    const size_t entryCount = sizeof(kSupportedFormatEntries) / sizeof(kSupportedFormatEntries[0]);



    AMediaFormatUtils::CopyFormatEntries(overlay, format, kSupportedFormatEntries, entryCount);

    return format;

    // ------- Copy parameters from source and options to the destination -------

    auto trackFormat = std::shared_ptr<AMediaFormat>(AMediaFormat_new(), &AMediaFormat_delete);

    AMediaFormatUtils::CopyFormatEntries(srcFormat, trackFormat.get(), srcParamsToCopy);

    AMediaFormatUtils::CopyFormatEntries(options, trackFormat.get(), kSupportedOptions);

    return trackFormat;

}



void MediaTranscoder::onThreadFinished(const void* thread, media_status_t threadStatus,
@@ -270,7 +284,8 @@ std::vector<std::shared_ptr<AMediaFormat>> MediaTranscoder::getTrackFormats() co 
    return trackFormats;

}



media_status_t MediaTranscoder::configureTrackFormat(size_t trackIndex, AMediaFormat* trackFormat) {

media_status_t MediaTranscoder::configureTrackFormat(size_t trackIndex,

                                                     AMediaFormat* destinationOptions) {

    if (mSampleReader == nullptr) {

        LOG(ERROR) << "Source must be configured before tracks";

        return AMEDIA_ERROR_INVALID_OPERATION;
@@ -281,14 +296,15 @@ media_status_t MediaTranscoder::configureTrackFormat(size_t trackIndex, AMediaFo 
    }



    std::shared_ptr<MediaTrackTranscoder> transcoder;

    std::shared_ptr<AMediaFormat> format;

    std::shared_ptr<AMediaFormat> trackFormat;



    if (trackFormat == nullptr) {

    if (destinationOptions == nullptr) {

        transcoder = std::make_shared<PassthroughTrackTranscoder>(shared_from_this());

    } else {

        AMediaFormat* srcTrackFormat = mSourceTrackFormats[trackIndex].get();



        const char* srcMime = nullptr;

        if (!AMediaFormat_getString(mSourceTrackFormats[trackIndex].get(), AMEDIAFORMAT_KEY_MIME,

                                    &srcMime)) {

        if (!AMediaFormat_getString(srcTrackFormat, AMEDIAFORMAT_KEY_MIME, &srcMime)) {

            LOG(ERROR) << "Source track #" << trackIndex << " has no mime type";

            return AMEDIA_ERROR_MALFORMED;

        }
@@ -301,7 +317,7 @@ media_status_t MediaTranscoder::configureTrackFormat(size_t trackIndex, AMediaFo 
        }



        const char* dstMime = nullptr;

        if (AMediaFormat_getString(trackFormat, AMEDIAFORMAT_KEY_MIME, &dstMime)) {

        if (AMediaFormat_getString(destinationOptions, AMEDIAFORMAT_KEY_MIME, &dstMime)) {

            if (strncmp(dstMime, "video/", 6) != 0) {

                LOG(ERROR) << "Unable to convert media types for track #" << trackIndex << ", from "

                           << srcMime << " to " << dstMime;
@@ -311,14 +327,11 @@ media_status_t MediaTranscoder::configureTrackFormat(size_t trackIndex, AMediaFo 


        transcoder = VideoTrackTranscoder::create(shared_from_this(), mPid, mUid);



        AMediaFormat* mergedFormat =

                mergeMediaFormats(mSourceTrackFormats[trackIndex].get(), trackFormat);

        if (mergedFormat == nullptr) {

            LOG(ERROR) << "Unable to merge source and destination formats";

        trackFormat = createVideoTrackFormat(srcTrackFormat, destinationOptions);

        if (trackFormat == nullptr) {

            LOG(ERROR) << "Unable to create video track format";

            return AMEDIA_ERROR_UNKNOWN;

        }



        format = std::shared_ptr<AMediaFormat>(mergedFormat, &AMediaFormat_delete);

    }



    media_status_t status = mSampleReader->selectTrack(trackIndex);
@@ -327,7 +340,7 @@ media_status_t MediaTranscoder::configureTrackFormat(size_t trackIndex, AMediaFo 
        return status;

    }



    status = transcoder->configure(mSampleReader, trackIndex, format);

    status = transcoder->configure(mSampleReader, trackIndex, trackFormat);

    if (status != AMEDIA_OK) {

        LOG(ERROR) << "Configure track transcoder for track #" << trackIndex << " returned error "

                   << status;

media/libmediatranscoding/transcoder/NdkCommon.cpp

+6 −6
Original line number Diff line number Diff line
@@ -60,19 +60,19 @@ DEFINE_FORMAT_VALUE_COPY_FUNC(int64_t, Int64); 
DEFINE_FORMAT_VALUE_COPY_FUNC(int32_t, Int32);

DEFINE_FORMAT_VALUE_COPY_FUNC(float, Float);



void CopyFormatEntries(AMediaFormat* from, AMediaFormat* to, const EntryCopier* entries,

                       size_t entryCount) {

void CopyFormatEntries(AMediaFormat* from, AMediaFormat* to,

                       const std::vector<EntryCopier>& entries) {

    if (from == nullptr || to == nullptr) {

        LOG(ERROR) << "Cannot copy null formats";

        return;

    } else if (entries == nullptr || entryCount < 1) {

    } else if (entries.empty()) {

        LOG(WARNING) << "No entries to copy";

        return;

    }



    for (size_t i = 0; i < entryCount; ++i) {

        if (!entries[i].copy(entries[i].key, from, to) && entries[i].copy2 != nullptr) {

            entries[i].copy2(entries[i].key, from, to);

    for (auto& entry : entries) {

        if (!entry.copy(entry.key, from, to) && entry.copy2 != nullptr) {

            entry.copy2(entry.key, from, to);

        }

    }

}

media/libmediatranscoding/transcoder/VideoTrackTranscoder.cpp

+8 −14
Original line number Diff line number Diff line
@@ -173,7 +173,7 @@ struct AsyncCodecCallbackDispatch { 
        if (auto transcoder = wrapper->getTranscoder()) {

            const bool isDecoder = codec == transcoder->mDecoder;

            const char* kCodecName = (isDecoder ? "Decoder" : "Encoder");

            LOG(DEBUG) << kCodecName << " format changed: " << AMediaFormat_toString(format);

            LOG(INFO) << kCodecName << " format changed: " << AMediaFormat_toString(format);

            transcoder->mCodecMessageQueue.push([transcoder, format, isDecoder] {

                transcoder->updateTrackFormat(format, isDecoder);

            });
@@ -306,7 +306,7 @@ media_status_t VideoTrackTranscoder::configureDestinationFormat( 
    }

    mEncoder = std::make_shared<CodecWrapper>(encoder, shared_from_this());



    LOG(DEBUG) << "Configuring encoder with: " << AMediaFormat_toString(mDestinationFormat.get());

    LOG(INFO) << "Configuring encoder with: " << AMediaFormat_toString(mDestinationFormat.get());

    status = AMediaCodec_configure(mEncoder->getCodec(), mDestinationFormat.get(),

                                   NULL /* surface */, NULL /* crypto */,

                                   AMEDIACODEC_CONFIGURE_FLAG_ENCODE);
@@ -358,15 +358,13 @@ media_status_t VideoTrackTranscoder::configureDestinationFormat( 
    AMediaFormat_setInt32(decoderFormat.get(), TBD_AMEDIACODEC_PARAMETER_KEY_ALLOW_FRAME_DROP, 0);



    // Copy over configurations that apply to both encoder and decoder.

    static const EntryCopier kEncoderEntriesToCopy[] = {

    static const std::vector<EntryCopier> kEncoderEntriesToCopy{

            ENTRY_COPIER2(AMEDIAFORMAT_KEY_OPERATING_RATE, Float, Int32),

            ENTRY_COPIER(AMEDIAFORMAT_KEY_PRIORITY, Int32),

    };

    const size_t entryCount = sizeof(kEncoderEntriesToCopy) / sizeof(kEncoderEntriesToCopy[0]);

    CopyFormatEntries(mDestinationFormat.get(), decoderFormat.get(), kEncoderEntriesToCopy,

                      entryCount);

    CopyFormatEntries(mDestinationFormat.get(), decoderFormat.get(), kEncoderEntriesToCopy);



    LOG(DEBUG) << "Configuring decoder with: " << AMediaFormat_toString(decoderFormat.get());

    LOG(INFO) << "Configuring decoder with: " << AMediaFormat_toString(decoderFormat.get());

    status = AMediaCodec_configure(mDecoder, decoderFormat.get(), mSurface, NULL /* crypto */,

                                   0 /* flags */);

    if (status != AMEDIA_OK) {
@@ -513,9 +511,6 @@ void VideoTrackTranscoder::dequeueOutputSample(int32_t bufferIndex, 
        onOutputSampleAvailable(sample);



        mLastSampleWasSync = sample->info.flags & SAMPLE_FLAG_SYNC_SAMPLE;

    } else if (bufferIndex == AMEDIACODEC_INFO_OUTPUT_FORMAT_CHANGED) {

        AMediaFormat* newFormat = AMediaCodec_getOutputFormat(mEncoder->getCodec());

        LOG(DEBUG) << "Encoder output format changed: " << AMediaFormat_toString(newFormat);

    }



    if (bufferInfo.flags & AMEDIACODEC_BUFFER_FLAG_END_OF_STREAM) {
@@ -535,15 +530,14 @@ void VideoTrackTranscoder::dequeueOutputSample(int32_t bufferIndex, 


void VideoTrackTranscoder::updateTrackFormat(AMediaFormat* outputFormat, bool fromDecoder) {

    if (fromDecoder) {

        static const AMediaFormatUtils::EntryCopier kValuesToCopy[] = {

        static const std::vector<AMediaFormatUtils::EntryCopier> kValuesToCopy{

                ENTRY_COPIER(AMEDIAFORMAT_KEY_COLOR_RANGE, Int32),

                ENTRY_COPIER(AMEDIAFORMAT_KEY_COLOR_STANDARD, Int32),

                ENTRY_COPIER(AMEDIAFORMAT_KEY_COLOR_TRANSFER, Int32),

        };

        AMediaFormat* params = AMediaFormat_new();

        if (params != nullptr) {

            AMediaFormatUtils::CopyFormatEntries(outputFormat, params, kValuesToCopy,

                                                 std::size(kValuesToCopy));

            AMediaFormatUtils::CopyFormatEntries(outputFormat, params, kValuesToCopy);

            if (AMediaCodec_setParameters(mEncoder->getCodec(), params) != AMEDIA_OK) {

                LOG(WARNING) << "Unable to update encoder with color information";

            }
@@ -615,7 +609,7 @@ void VideoTrackTranscoder::updateTrackFormat(AMediaFormat* outputFormat, bool fr 
    // TODO: transfer other fields as required.



    mActualOutputFormat = std::shared_ptr<AMediaFormat>(formatCopy, &AMediaFormat_delete);

    LOG(DEBUG) << "Actual output format: " << AMediaFormat_toString(formatCopy);

    LOG(INFO) << "Actual output format: " << AMediaFormat_toString(formatCopy);



    notifyTrackFormatAvailable();

}

media/libmediatranscoding/transcoder/include/media/NdkCommon.h

+4 −2
Original line number Diff line number Diff line
@@ -19,6 +19,8 @@ 


#include <media/NdkMediaFormat.h>



#include <vector>



extern const char* AMEDIA_MIMETYPE_VIDEO_VP8;

extern const char* AMEDIA_MIMETYPE_VIDEO_VP9;

extern const char* AMEDIA_MIMETYPE_VIDEO_AV1;
@@ -82,8 +84,8 @@ bool CopyFormatEntryInt64(const char* key, AMediaFormat* from, AMediaFormat* to) 
bool CopyFormatEntryInt32(const char* key, AMediaFormat* from, AMediaFormat* to);

bool CopyFormatEntryFloat(const char* key, AMediaFormat* from, AMediaFormat* to);



void CopyFormatEntries(AMediaFormat* from, AMediaFormat* to, const EntryCopier* entries,

                       size_t entryCount);

void CopyFormatEntries(AMediaFormat* from, AMediaFormat* to,

                       const std::vector<EntryCopier>& entries);



bool SetDefaultFormatValueFloat(const char* key, AMediaFormat* format, float value);

bool SetDefaultFormatValueInt32(const char* key, AMediaFormat* format, int32_t value);