Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 90fcf68f authored by Lajos Molnar's avatar Lajos Molnar
Browse files

stagefright: add support for output frame rendered callback 

- Added FRAME_RENDERED event in OMX, used by tunneled video decoders
  to signal rendered event timing
- Track buffers sent for rendering in ACodec and in SoftwareRenderer, and
  determine when they have rendered
- Propagate render times to MediaCodec

Bug: 20503131
Change-Id: Idf0a8714d5368b237c2285dd39fa82db847c232f
parent 9b132a7b

include/media/IOMX.h

+6 −1
Original line number Diff line number Diff line
@@ -198,7 +198,7 @@ struct omx_message { 
        EVENT,

        EMPTY_BUFFER_DONE,

        FILL_BUFFER_DONE,



        FRAME_RENDERED,

    } type;



    IOMX::node_id node;
@@ -226,6 +226,11 @@ struct omx_message { 
            OMX_TICKS timestamp;

        } extended_buffer_data;



        // if type == FRAME_RENDERED

        struct {

            OMX_TICKS timestamp;

            OMX_S64 nanoTime;

        } render_data;

    } u;

};

include/media/stagefright/ACodec.h

+20 −0
Original line number Diff line number Diff line
@@ -24,6 +24,7 @@ 
#include <media/IOMX.h>

#include <media/stagefright/foundation/AHierarchicalStateMachine.h>

#include <media/stagefright/CodecBase.h>

#include <media/stagefright/FrameRenderTracker.h>

#include <media/stagefright/SkipCutBuffer.h>

#include <OMX_Audio.h>
@@ -162,6 +163,7 @@ private: 
        sp<ABuffer> mData;

        sp<GraphicBuffer> mGraphicBuffer;

        int mFenceFd;

        FrameRenderTracker::Info *mRenderInfo;



        // The following field and 4 methods are used for debugging only

        bool mIsReadFence;
@@ -214,6 +216,7 @@ private: 
    sp<AMessage> mOutputFormat;

    sp<AMessage> mBaseOutputFormat;



    FrameRenderTracker mRenderTracker; // render information for buffers rendered by ACodec

    Vector<BufferInfo> mBuffers[2];

    bool mPortEOS[2];

    status_t mInputEOSResult;
@@ -375,6 +378,23 @@ private: 
    void deferMessage(const sp<AMessage> &msg);

    void processDeferredMessages();



    void onFrameRendered(int64_t mediaTimeUs, nsecs_t systemNano);

    // called when we have dequeued a buffer |buf| from the native window to track render info.

    // |fenceFd| is the dequeue fence, and |info| points to the buffer info where this buffer is

    // stored.

    void updateRenderInfoForDequeuedBuffer(

            ANativeWindowBuffer *buf, int fenceFd, BufferInfo *info);



    // Checks to see if any frames have rendered up until |until|, and to notify client

    // (MediaCodec) of rendered frames up-until the frame pointed to by |until| or the first

    // unrendered frame. These frames are removed from the render queue.

    // If |dropIncomplete| is true, unrendered frames up-until |until| will be dropped from the

    // queue, allowing all rendered framed up till then to be notified of.

    // (This will effectively clear the render queue up-until (and including) |until|.)

    // If |until| is NULL, or is not in the rendered queue, this method will check all frames.

    void notifyOfRenderedFrames(

            bool dropIncomplete = false, FrameRenderTracker::Info *until = NULL);



    void sendFormatChange(const sp<AMessage> &reply);

    status_t getPortFormat(OMX_U32 portIndex, sp<AMessage> &notify);

include/media/stagefright/CodecBase.h

+1 −0
Original line number Diff line number Diff line
@@ -43,6 +43,7 @@ struct CodecBase : public AHandler { 
        kWhatInputSurfaceAccepted = 'isfa',

        kWhatSignaledInputEOS    = 'seos',

        kWhatBuffersAllocated    = 'allc',

        kWhatOutputFramesRendered = 'outR',

    };



    virtual void setNotificationMessage(const sp<AMessage> &msg) = 0;

include/media/stagefright/FrameRenderTracker.h

0 → 100644
+142 −0
Original line number Diff line number Diff line 
/*

 * Copyright 2015 The Android Open Source Project

 *

 * Licensed under the Apache License, Version 2.0 (the "License");

 * you may not use this file except in compliance with the License.

 * You may obtain a copy of the License at

 *

 *      http://www.apache.org/licenses/LICENSE-2.0

 *

 * Unless required by applicable law or agreed to in writing, software

 * distributed under the License is distributed on an "AS IS" BASIS,

 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

 * See the License for the specific language governing permissions and

 * limitations under the License.

 */



#ifndef FRAME_RENDER_TRACKER_H_



#define FRAME_RENDER_TRACKER_H_



#include <utils/RefBase.h>

#include <utils/Timers.h>

#include <system/window.h>



#include <media/stagefright/foundation/ADebug.h>

#include <media/stagefright/foundation/AString.h>



#include <list>



namespace android {



class Fence;

class GraphicBuffer;



struct FrameRenderTracker : public RefBase {

    // Tracks the render information about a frame. Frames go through several states while

    // the render information is tracked:

    //

    // 1. queued frame: mMediaTime and mGraphicBuffer are set for the frame. mFence is the

    // queue fence (read fence). mIndex is negative, and mRenderTimeNs is invalid.

    // Key characteristics: mFence is not NULL and mIndex is negative.

    //

    // 2. dequeued frame: mFence is updated with the dequeue fence (write fence). mIndex is set.

    // Key characteristics: mFence is not NULL and mIndex is non-negative. mRenderTime is still

    // invalid.

    //

    // 3. rendered frame or frame: mFence is cleared, mRenderTimeNs is set.

    // Key characteristics: mFence is NULL.

    //

    struct Info {

        // set by client during onFrameQueued or onFrameRendered

        int64_t getMediaTimeUs() const  { return mMediaTimeUs; }



        // -1 if frame is not yet rendered

        nsecs_t getRenderTimeNs() const { return mRenderTimeNs; }



        // set by client during updateRenderInfoForDequeuedBuffer; -1 otherwise

        ssize_t getIndex() const        { return mIndex; }



        // creates information for a queued frame

        Info(int64_t mediaTimeUs, const sp<GraphicBuffer> &graphicBuffer, const sp<Fence> &fence)

            : mMediaTimeUs(mediaTimeUs),

              mRenderTimeNs(-1),

              mIndex(-1),

              mGraphicBuffer(graphicBuffer),

              mFence(fence) {

        }



        // creates information for a frame rendered on a tunneled surface

        Info(int64_t mediaTimeUs, nsecs_t renderTimeNs)

            : mMediaTimeUs(mediaTimeUs),

              mRenderTimeNs(renderTimeNs),

              mIndex(-1),

              mGraphicBuffer(NULL),

              mFence(NULL) {

        }



    private:

        int64_t mMediaTimeUs;

        nsecs_t mRenderTimeNs;

        ssize_t mIndex;         // to be used by client

        sp<GraphicBuffer> mGraphicBuffer;

        sp<Fence> mFence;



        friend class FrameRenderTracker;

    };



    FrameRenderTracker();



    void setComponentName(const AString &componentName);



    // clears all tracked frames, and resets last render time

    void clear(nsecs_t lastRenderTimeNs);



    // called when |graphicBuffer| corresponding to |mediaTimeUs| is

    // queued to the output surface using |fence|.

    void onFrameQueued(

            int64_t mediaTimeUs, const sp<GraphicBuffer> &graphicBuffer, const sp<Fence> &fence);



    // Called when we have dequeued a buffer |buf| from the native window to track render info.

    // |fenceFd| is the dequeue fence, and |index| is a positive buffer ID to be usable by the

    // client to track this render info among the dequeued buffers.

    // Returns pointer to the tracked info, or NULL if buffer is not tracked or if |index|

    // is negative.

    Info *updateInfoForDequeuedBuffer(ANativeWindowBuffer *buf, int fenceFd, int index);



    // called when tunneled codec signals frame rendered event

    // returns BAD_VALUE if systemNano is not monotonic. Otherwise, returns OK.

    status_t onFrameRendered(int64_t mediaTimeUs, nsecs_t systemNano);



    // Checks to see if any frames have rendered up until |until|. If |until| is NULL or not a

    // tracked info, this method searches the entire render queue.

    // Returns list of rendered frames up-until the frame pointed to by |until| or the first

    // unrendered frame, as well as any dropped frames (those with invalid fence) up-until |until|.

    // These frames are removed from the render queue.

    // If |dropIncomplete| is true, unrendered frames up-until |until| will also be dropped from the

    // queue, allowing all rendered framed up till then to be notified of.

    // (This will effectively clear the render queue up-until (and including) |until|.)

    std::list<Info> checkFencesAndGetRenderedFrames(const Info *until, bool dropIncomplete);



    // Stop tracking a queued frame (e.g. if the frame has been discarded). If |info| is NULL or is

    // not tracked, this method is a no-op.

    void untrackFrame(const Info *info);



    void dumpRenderQueue() const;



    virtual ~FrameRenderTracker();



private:



    // Render information for buffers. Regular surface buffers are queued in the order of

    // rendering. Tunneled buffers are queued in the order of receipt.

    std::list<Info> mRenderQueue;

    nsecs_t mLastRenderTimeNs;

    AString mComponentName;



    DISALLOW_EVIL_CONSTRUCTORS(FrameRenderTracker);

};



}  // namespace android



#endif  // FRAME_RENDER_TRACKER_H_

include/media/stagefright/MediaCodec.h

+12 −0
Original line number Diff line number Diff line
@@ -22,6 +22,7 @@ 
#include <media/hardware/CryptoAPI.h>

#include <media/MediaResource.h>

#include <media/stagefright/foundation/AHandler.h>

#include <media/stagefright/FrameRenderTracker.h>

#include <utils/Vector.h>



namespace android {
@@ -76,6 +77,8 @@ struct MediaCodec : public AHandler { 


    status_t setCallback(const sp<AMessage> &callback);



    status_t setOnFrameRenderedNotification(const sp<AMessage> &notify);



    status_t createInputSurface(sp<IGraphicBufferProducer>* bufferProducer);



    status_t setInputSurface(const sp<PersistentSurface> &surface);
@@ -157,6 +160,12 @@ struct MediaCodec : public AHandler { 


    status_t setParameters(const sp<AMessage> &params);



    // Create a MediaCodec notification message from a list of rendered or dropped render infos

    // by adding rendered frame information to a base notification message. Returns the number

    // of frames that were rendered.

    static size_t CreateFramesRenderedMessage(

            std::list<FrameRenderTracker::Info> done, sp<AMessage> &msg);



protected:

    virtual ~MediaCodec();

    virtual void onMessageReceived(const sp<AMessage> &msg);
@@ -212,6 +221,7 @@ private: 
        kWhatGetName                        = 'getN',

        kWhatSetParameters                  = 'setP',

        kWhatSetCallback                    = 'setC',

        kWhatSetNotification                = 'setN',

    };



    enum {
@@ -275,9 +285,11 @@ private: 
    status_t mStickyError;

    sp<Surface> mSurface;

    SoftwareRenderer *mSoftRenderer;



    sp<AMessage> mOutputFormat;

    sp<AMessage> mInputFormat;

    sp<AMessage> mCallback;

    sp<AMessage> mOnFrameRenderedNotification;

    sp<MemoryDealer> mDealer;



    sp<IResourceManagerClient> mResourceManagerClient;