Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 94ea60f9 authored by Richard Fitzgerald's avatar Richard Fitzgerald Committed by Eric Laurent
Browse files

stagefright: offload playback support 



Offloading of compressed audio decoding to audio DSP
is implemented for audio only, non streamed content.
when the datasource is

AudioPlayer:
- Create an offloaded sink when playing a compressed
source
- Send metadata to audio HAL
- Return sink start error to AwesomePlayer so that a
new player for PCM audio can be created in case of problem.
- Forward stream end and tear down callback events to AwesomePlayer
- Stop the sink and wait for stream end callback when EOS is reached.
- Pause and restart the sink if needed before flushing when seeking
(otherwise flush is a no op).
- For current media time, directly query the render position from the
sink and offset by the start position (seek to time)

AwesomePlayer:

- When initializing the audio decoder, check with audio policy manager
if offloading is supported. If yes, create the software decoder in
case a reconfiguration is needed but connect the audio track directly
to the AudioPlayer.
- In case of error when starting the AudioPlayer, reconnect the software
decoder (OMXSource) and recreate a PCM AudioPlayer.
- Handle AudioPlayer tear down event by detroying and recreating the
AudioPlayer to allow transitions between situations were offloading
is supported or not.
- Force tear down of offloaded AudioPlayer when paused for a certain time:
This will close the sink and allow the DSP to power down.

Utils:
- Added helper methods:
    - send meta data to audio ia sink setParameters
    - query audio policy manager if offloading is supported for a
given audio content

Change-Id: I115842ce424f947b966d45e253a74d3fd5df9aae
Signed-off-by: default avatarEric Laurent <elaurent@google.com>
parent d89532e1

include/media/stagefright/AudioPlayer.h

+12 −4
Original line number Diff line number Diff line
@@ -38,7 +38,10 @@ public: 


    enum {

        ALLOW_DEEP_BUFFERING = 0x01,

        USE_OFFLOAD = 0x02

        USE_OFFLOAD = 0x02,

        HAS_VIDEO   = 0x1000,

        IS_STREAMING = 0x2000



    };



    AudioPlayer(const sp<MediaPlayerBase::AudioSink> &audioSink,
@@ -56,7 +59,7 @@ public: 
    status_t start(bool sourceAlreadyStarted = false);



    void pause(bool playPendingSamples = false);

    void resume();

    status_t resume();



    // Returns the timestamp of the last buffer played (in us).

    int64_t getMediaTimeUs();
@@ -104,11 +107,13 @@ private: 
    MediaBuffer *mFirstBuffer;



    sp<MediaPlayerBase::AudioSink> mAudioSink;

    bool mAllowDeepBuffering;       // allow audio deep audio buffers. Helps with low power audio

                                    // playback but implies high latency

    AwesomePlayer *mObserver;

    int64_t mPinnedTimeUs;



    bool mPlaying;

    int64_t mStartPosUs;

    const uint32_t mCreateFlags;



    static void AudioCallback(int event, void *user, void *info);

    void AudioCallback(int event, void *info);
@@ -126,6 +131,9 @@ private: 
    uint32_t getNumFramesPendingPlayout() const;

    int64_t getOutputPlayPositionUs_l() const;



    bool allowDeepBuffering() const { return (mCreateFlags & ALLOW_DEEP_BUFFERING) != 0; }

    bool useOffload() const { return (mCreateFlags & USE_OFFLOAD) != 0; }



    AudioPlayer(const AudioPlayer &);

    AudioPlayer &operator=(const AudioPlayer &);

};

libvideoeditor/lvpp/VideoEditorAudioPlayer.cpp

+2 −1
Original line number Diff line number Diff line
@@ -149,7 +149,7 @@ void VideoEditorAudioPlayer::clear() { 
    mStarted = false;

}



void VideoEditorAudioPlayer::resume() {

status_t VideoEditorAudioPlayer::resume() {

    ALOGV("resume");



    AudioMixSettings audioMixSettings;
@@ -180,6 +180,7 @@ void VideoEditorAudioPlayer::resume() { 
    } else {

        mAudioTrack->start();

    }

    return OK;

}



status_t VideoEditorAudioPlayer::seekTo(int64_t time_us) {

libvideoeditor/lvpp/VideoEditorAudioPlayer.h

+1 −1
Original line number Diff line number Diff line
@@ -58,7 +58,7 @@ public: 


    status_t start(bool sourceAlreadyStarted = false);

    void pause(bool playPendingSamples = false);

    void resume();

    status_t resume();

    status_t seekTo(int64_t time_us);

    bool isSeeking();

    bool reachedEOS(status_t *finalStatus);

media/libstagefright/Android.mk

+1 −0
Original line number Diff line number Diff line
@@ -100,6 +100,7 @@ LOCAL_STATIC_LIBRARIES := \ 
        libstagefright_mpeg2ts \

        libstagefright_id3 \

        libFLAC \

        libmedia_helper



LOCAL_SRC_FILES += \

        chromium_http_stub.cpp

media/libstagefright/AudioPlayer.cpp

+281 −67
Original line number Diff line number Diff line
@@ -17,6 +17,7 @@ 
//#define LOG_NDEBUG 0

#define LOG_TAG "AudioPlayer"

#include <utils/Log.h>

#include <cutils/compiler.h>



#include <binder/IPCThreadState.h>

#include <media/AudioTrack.h>
@@ -27,6 +28,7 @@ 
#include <media/stagefright/MediaErrors.h>

#include <media/stagefright/MediaSource.h>

#include <media/stagefright/MetaData.h>

#include <media/stagefright/Utils.h>



#include "include/AwesomePlayer.h"
@@ -47,14 +49,17 @@ AudioPlayer::AudioPlayer( 
      mSeeking(false),

      mReachedEOS(false),

      mFinalStatus(OK),

      mSeekTimeUs(0),

      mStarted(false),

      mIsFirstBuffer(false),

      mFirstBufferResult(OK),

      mFirstBuffer(NULL),

      mAudioSink(audioSink),

      mAllowDeepBuffering((flags & ALLOW_DEEP_BUFFERING) != 0),

      mObserver(observer),

      mPinnedTimeUs(-1ll) {

      mPinnedTimeUs(-1ll),

      mPlaying(false),

      mStartPosUs(0),

      mCreateFlags(flags) {

}



AudioPlayer::~AudioPlayer() {
@@ -109,7 +114,7 @@ status_t AudioPlayer::start(bool sourceAlreadyStarted) { 
    const char *mime;

    bool success = format->findCString(kKeyMIMEType, &mime);

    CHECK(success);

    CHECK(!strcasecmp(mime, MEDIA_MIMETYPE_AUDIO_RAW));

    CHECK(useOffload() || !strcasecmp(mime, MEDIA_MIMETYPE_AUDIO_RAW));



    success = format->findInt32(kKeySampleRate, &mSampleRate);

    CHECK(success);
@@ -125,16 +130,74 @@ status_t AudioPlayer::start(bool sourceAlreadyStarted) { 
        channelMask = CHANNEL_MASK_USE_CHANNEL_ORDER;

    }



    audio_format_t audioFormat = AUDIO_FORMAT_PCM_16_BIT;



    if (useOffload()) {

        if (mapMimeToAudioFormat(audioFormat, mime) != OK) {

            ALOGE("Couldn't map mime type \"%s\" to a valid AudioSystem::audio_format", mime);

            audioFormat = AUDIO_FORMAT_INVALID;

        } else {

            ALOGV("Mime type \"%s\" mapped to audio_format 0x%x", mime, audioFormat);

        }

    }



    int avgBitRate = -1;

    format->findInt32(kKeyBitRate, &avgBitRate);



    if (mAudioSink.get() != NULL) {



        uint32_t flags = AUDIO_OUTPUT_FLAG_NONE;

        audio_offload_info_t offloadInfo = AUDIO_INFO_INITIALIZER;



        if (allowDeepBuffering()) {

            flags |= AUDIO_OUTPUT_FLAG_DEEP_BUFFER;

        }

        if (useOffload()) {

            flags |= AUDIO_OUTPUT_FLAG_COMPRESS_OFFLOAD;



            int64_t durationUs;

            if (format->findInt64(kKeyDuration, &durationUs)) {

                offloadInfo.duration_us = durationUs;

            } else {

                offloadInfo.duration_us = -1;

            }



            offloadInfo.sample_rate = mSampleRate;

            offloadInfo.channel_mask = channelMask;

            offloadInfo.format = audioFormat;

            offloadInfo.stream_type = AUDIO_STREAM_MUSIC;

            offloadInfo.bit_rate = avgBitRate;

            offloadInfo.has_video = ((mCreateFlags & HAS_VIDEO) != 0);

            offloadInfo.is_streaming = ((mCreateFlags & IS_STREAMING) != 0);

        }



        status_t err = mAudioSink->open(

                mSampleRate, numChannels, channelMask, AUDIO_FORMAT_PCM_16_BIT,

                mSampleRate, numChannels, channelMask, audioFormat,

                DEFAULT_AUDIOSINK_BUFFERCOUNT,

                &AudioPlayer::AudioSinkCallback,

                this,

                (mAllowDeepBuffering ?

                            AUDIO_OUTPUT_FLAG_DEEP_BUFFER :

                            AUDIO_OUTPUT_FLAG_NONE));

                (audio_output_flags_t)flags,

                useOffload() ? &offloadInfo : NULL);



        if (err == OK) {

            mLatencyUs = (int64_t)mAudioSink->latency() * 1000;

            mFrameSize = mAudioSink->frameSize();



            if (useOffload()) {

                // If the playback is offloaded to h/w we pass the

                // HAL some metadata information

                // We don't want to do this for PCM because it will be going

                // through the AudioFlinger mixer before reaching the hardware

                sendMetaDataToHal(mAudioSink, format);

            }



            err = mAudioSink->start();

            // do not alter behavior for non offloaded tracks: ignore start status.

            if (!useOffload()) {

                err = OK;

            }

        }



        if (err != OK) {

            if (mFirstBuffer != NULL) {

                mFirstBuffer->release();
@@ -148,10 +211,6 @@ status_t AudioPlayer::start(bool sourceAlreadyStarted) { 
            return err;

        }



        mLatencyUs = (int64_t)mAudioSink->latency() * 1000;

        mFrameSize = mAudioSink->frameSize();



        mAudioSink->start();

    } else {

        // playing to an AudioTrack, set up mask if necessary

        audio_channel_mask_t audioMask = channelMask == CHANNEL_MASK_USE_CHANNEL_ORDER ?
@@ -186,6 +245,7 @@ status_t AudioPlayer::start(bool sourceAlreadyStarted) { 
    }



    mStarted = true;

    mPlaying = true;

    mPinnedTimeUs = -1ll;



    return OK;
@@ -212,27 +272,56 @@ void AudioPlayer::pause(bool playPendingSamples) { 


        mPinnedTimeUs = ALooper::GetNowUs();

    }



    mPlaying = false;

}



void AudioPlayer::resume() {

status_t AudioPlayer::resume() {

    CHECK(mStarted);

    status_t err;



    if (mAudioSink.get() != NULL) {

        mAudioSink->start();

        err = mAudioSink->start();

    } else {

        mAudioTrack->start();

        err = mAudioTrack->start();

    }



    if (err == OK) {

        mPlaying = true;

    }



    return err;

}



void AudioPlayer::reset() {

    CHECK(mStarted);



    ALOGV("reset: mPlaying=%d mReachedEOS=%d useOffload=%d",

                                mPlaying, mReachedEOS, useOffload() );



    if (mAudioSink.get() != NULL) {

        mAudioSink->stop();

        // If we're closing and have reached EOS, we don't want to flush

        // the track because if it is offloaded there could be a small

        // amount of residual data in the hardware buffer which we must

        // play to give gapless playback.

        // But if we're resetting when paused or before we've reached EOS

        // we can't be doing a gapless playback and there could be a large

        // amount of data queued in the hardware if the track is offloaded,

        // so we must flush to prevent a track switch being delayed playing

        // the buffered data that we don't want now

        if (!mPlaying || !mReachedEOS) {

            mAudioSink->flush();

        }



        mAudioSink->close();

    } else {

        mAudioTrack->stop();



        if (!mPlaying || !mReachedEOS) {

            mAudioTrack->flush();

        }



        mAudioTrack.clear();

    }
@@ -256,11 +345,17 @@ void AudioPlayer::reset() { 
    // The following hack is necessary to ensure that the OMX

    // component is completely released by the time we may try

    // to instantiate it again.

    // When offloading, the OMX component is not used so this hack

    // is not needed

    if (!useOffload()) {

        wp<MediaSource> tmp = mSource;

        mSource.clear();

        while (tmp.promote() != NULL) {

            usleep(1000);

        }

    } else {

        mSource.clear();

    }

    IPCThreadState::self()->flushCommands();



    mNumFramesPlayed = 0;
@@ -271,6 +366,8 @@ void AudioPlayer::reset() { 
    mReachedEOS = false;

    mFinalStatus = OK;

    mStarted = false;

    mPlaying = false;

    mStartPosUs = 0;

}



// static
@@ -291,6 +388,15 @@ bool AudioPlayer::reachedEOS(status_t *finalStatus) { 
    return mReachedEOS;

}



void AudioPlayer::notifyAudioEOS() {

    ALOGV("AudioPlayer@0x%p notifyAudioEOS", this);



    if (mObserver != NULL) {

        mObserver->postAudioEOS(0);

        ALOGV("Notified observer of EOS!");

    }

}



status_t AudioPlayer::setPlaybackRatePermille(int32_t ratePermille) {

    if (mAudioSink.get() != NULL) {

        return mAudioSink->setPlaybackRatePermille(ratePermille);
@@ -308,19 +414,41 @@ size_t AudioPlayer::AudioSinkCallback( 
        MediaPlayerBase::AudioSink::cb_event_t event) {

    AudioPlayer *me = (AudioPlayer *)cookie;



    switch(event) {

    case MediaPlayerBase::AudioSink::CB_EVENT_FILL_BUFFER:

        return me->fillBuffer(buffer, size);



    case MediaPlayerBase::AudioSink::CB_EVENT_STREAM_END:

        ALOGV("AudioSinkCallback: stream end");

        me->mReachedEOS = true;

        me->notifyAudioEOS();

        break;



    case MediaPlayerBase::AudioSink::CB_EVENT_TEAR_DOWN:

        ALOGV("AudioSinkCallback: Tear down event");

        me->mObserver->postAudioTearDown();

        break;

    }



void AudioPlayer::AudioCallback(int event, void *info) {

    if (event != AudioTrack::EVENT_MORE_DATA) {

        return;

    return 0;

}



void AudioPlayer::AudioCallback(int event, void *info) {

    switch (event) {

    case AudioTrack::EVENT_MORE_DATA:

        {

        AudioTrack::Buffer *buffer = (AudioTrack::Buffer *)info;

        size_t numBytesWritten = fillBuffer(buffer->raw, buffer->size);



        buffer->size = numBytesWritten;

        }

        break;



    case AudioTrack::EVENT_STREAM_END:

        mReachedEOS = true;

        notifyAudioEOS();

        break;

    }

}



uint32_t AudioPlayer::getNumFramesPendingPlayout() const {

    uint32_t numFramesPlayedOut;
@@ -359,6 +487,7 @@ size_t AudioPlayer::fillBuffer(void *data, size_t size) { 
    size_t size_remaining = size;

    while (size_remaining > 0) {

        MediaSource::ReadOptions options;

        bool refreshSeekTime = false;



        {

            Mutex::Autolock autoLock(mLock);
@@ -373,6 +502,7 @@ size_t AudioPlayer::fillBuffer(void *data, size_t size) { 
                }



                options.setSeekTo(mSeekTimeUs);

                refreshSeekTime = true;



                if (mInputBuffer != NULL) {

                    mInputBuffer->release();
@@ -405,7 +535,17 @@ size_t AudioPlayer::fillBuffer(void *data, size_t size) { 
            Mutex::Autolock autoLock(mLock);



            if (err != OK) {

                if (mObserver && !mReachedEOS) {

                if (!mReachedEOS) {

                    if (useOffload()) {

                        // no more buffers to push - stop() and wait for STREAM_END

                        // don't set mReachedEOS until stream end received

                        if (mAudioSink != NULL) {

                            mAudioSink->stop();

                        } else {

                            mAudioTrack->stop();

                        }

                    } else {

                        if (mObserver) {

                            // We don't want to post EOS right away but only

                            // after all frames have actually been played out.
@@ -442,6 +582,9 @@ size_t AudioPlayer::fillBuffer(void *data, size_t size) { 
                        }



                        mReachedEOS = true;

                    }

                }



                mFinalStatus = err;

                break;

            }
@@ -452,19 +595,36 @@ size_t AudioPlayer::fillBuffer(void *data, size_t size) { 
                mLatencyUs = (int64_t)mAudioTrack->latency() * 1000;

            }



            if(mInputBuffer->range_length() != 0) {

                CHECK(mInputBuffer->meta_data()->findInt64(

                        kKeyTime, &mPositionTimeMediaUs));

            }



            // need to adjust the mStartPosUs for offload decoding since parser

            // might not be able to get the exact seek time requested.

            if (refreshSeekTime && useOffload()) {

                if (postSeekComplete) {

                    ALOGV("fillBuffer is going to post SEEK_COMPLETE");

                    mObserver->postAudioSeekComplete();

                    postSeekComplete = false;

                }



                mStartPosUs = mPositionTimeMediaUs;

                ALOGV("adjust seek time to: %.2f", mStartPosUs/ 1E6);

            }



            if (!useOffload()) {

                mPositionTimeRealUs =

                    ((mNumFramesPlayed + size_done / mFrameSize) * 1000000)

                        / mSampleRate;



                ALOGV("buffer->size() = %d, "

                     "mPositionTimeMediaUs=%.2f mPositionTimeRealUs=%.2f",

                     mInputBuffer->range_length(),

                     mPositionTimeMediaUs / 1E6, mPositionTimeRealUs / 1E6);

            }



        }



        if (mInputBuffer->range_length() == 0) {

            mInputBuffer->release();

            mInputBuffer = NULL;
@@ -488,6 +648,13 @@ size_t AudioPlayer::fillBuffer(void *data, size_t size) { 
        size_remaining -= copy;

    }



    if (useOffload()) {

        // We must ask the hardware what it has played

        mPositionTimeRealUs = getOutputPlayPositionUs_l();

        ALOGV("mPositionTimeMediaUs=%.2f mPositionTimeRealUs=%.2f",

             mPositionTimeMediaUs / 1E6, mPositionTimeRealUs / 1E6);

    }



    {

        Mutex::Autolock autoLock(mLock);

        mNumFramesPlayed += size_done / mFrameSize;
@@ -536,9 +703,36 @@ int64_t AudioPlayer::getRealTimeUsLocked() const { 
    return result + diffUs;

}



int64_t AudioPlayer::getOutputPlayPositionUs_l() const

{

    uint32_t playedSamples = 0;

    if (mAudioSink != NULL) {

        mAudioSink->getPosition(&playedSamples);

    } else {

        mAudioTrack->getPosition(&playedSamples);

    }



    const int64_t playedUs = (static_cast<int64_t>(playedSamples) * 1000000 ) / mSampleRate;



    // HAL position is relative to the first buffer we sent at mStartPosUs

    const int64_t renderedDuration = mStartPosUs + playedUs;

    ALOGV("getOutputPlayPositionUs_l %lld", renderedDuration);

    return renderedDuration;

}



int64_t AudioPlayer::getMediaTimeUs() {

    Mutex::Autolock autoLock(mLock);



    if (useOffload()) {

        if (mSeeking) {

            return mSeekTimeUs;

        }

        mPositionTimeRealUs = getOutputPlayPositionUs_l();

        ALOGV("getMediaTimeUs getOutputPlayPositionUs_l() mPositionTimeRealUs %lld",

              mPositionTimeRealUs);

        return mPositionTimeRealUs;

    }



    if (mPositionTimeMediaUs < 0 || mPositionTimeRealUs < 0) {

        if (mSeeking) {

            return mSeekTimeUs;
@@ -547,6 +741,11 @@ int64_t AudioPlayer::getMediaTimeUs() { 
        return 0;

    }



    if (useOffload()) {

        mPositionTimeRealUs = getOutputPlayPositionUs_l();

        return mPositionTimeRealUs;

    }



    int64_t realTimeOffset = getRealTimeUsLocked() - mPositionTimeRealUs;

    if (realTimeOffset < 0) {

        realTimeOffset = 0;
@@ -568,19 +767,34 @@ bool AudioPlayer::getMediaTimeMapping( 
status_t AudioPlayer::seekTo(int64_t time_us) {

    Mutex::Autolock autoLock(mLock);



    ALOGV("seekTo( %lld )", time_us);



    mSeeking = true;

    mPositionTimeRealUs = mPositionTimeMediaUs = -1;

    mReachedEOS = false;

    mSeekTimeUs = time_us;

    mStartPosUs = time_us;



    // Flush resets the number of played frames

    mNumFramesPlayed = 0;

    mNumFramesPlayedSysTimeUs = ALooper::GetNowUs();



    if (mAudioSink != NULL) {

        if (mPlaying) {

            mAudioSink->pause();

        }

        mAudioSink->flush();

        if (mPlaying) {

            mAudioSink->start();

        }

    } else {

        if (mPlaying) {

            mAudioTrack->pause();

        }

        mAudioTrack->flush();

        if (mPlaying) {

            mAudioTrack->start();

        }

    }



    return OK;