InputSurface: update InputSurfaceSource

        "Configurable.cpp",

        "InputBufferManager.cpp",

        "ParamTypes.cpp",

        "inputsurface/FrameDropper.cpp",

        "inputsurface/InputSurface.cpp",

        "inputsurface/InputSurfaceConnection.cpp",

        "inputsurface/InputSurfaceSource.cpp",

    ],

    header_libs: [

        "libmediandk",

        "libstagefright_aidl_bufferpool2",

        "libstagefright_bufferpool@2.0.1",

        "libstagefright_foundation",

        "libui",

        "libutils",

    ],

/*

 * Copyright (C) 2015 The Android Open Source Project

 * Copyright (C) 2024 The Android Open Source Project

 *

 * Licensed under the Apache License, Version 2.0 (the "License");

 * you may not use this file except in compliance with the License.

 * limitations under the License.

 */

#ifndef FRAME_DROPPER_H_

#define FRAME_DROPPER_H_

#pragma once

#include <utils/Errors.h>

#include <utils/RefBase.h>

#include <media/stagefright/foundation/ABase.h>

namespace android {

#include <C2.h>

namespace aidl::android::hardware::media::c2::implementation {

struct FrameDropper : public RefBase {

    // No frames will be dropped until a valid max frame rate is set.

/**

 * The class decides whether to drop a frame or not for InputSurface and

 * InputSurfaceConnection.

 */

struct FrameDropper {

    FrameDropper();

    // maxFrameRate required to be positive.

    // maxFrameRate negative causes shouldDrop() to always return false

    // maxFrameRate == 0 is illegal

    status_t setMaxFrameRate(float maxFrameRate);

    ~FrameDropper();

    /**

     * Sets max frame rate, which is based on for deciding frame drop.

     *

     * @param[in] maxFrameRate  negative value means there is no drop

     *                          zero value is ignored

     */

    void setMaxFrameRate(float maxFrameRate);

    // Returns false if max frame rate has not been set via setMaxFrameRate.

    /** Returns false if max frame rate has not been set via setMaxFrameRate. */

    bool shouldDrop(int64_t timeUs);

    // Returns true if all frame drop logic should be disabled.

    /** Returns true if all frame drop logic should be disabled. */

    bool disabled() { return (mMinIntervalUs == -1ll); }

protected:

    virtual ~FrameDropper();

private:

    int64_t mDesiredMinTimeUs;

    int64_t mMinIntervalUs;

    DISALLOW_EVIL_CONSTRUCTORS(FrameDropper);

};

}  // namespace android

#endif  // FRAME_DROPPER_H_

}  // namespace aidl::android::hardware::media::c2::implementation

/*

 * Copyright (C) 2013 The Android Open Source Project

 * Copyright (C) 2024 The Android Open Source Project

 *

 * Licensed under the Apache License, Version 2.0 (the "License");

 * you may not use this file except in compliance with the License.

 * limitations under the License.

 */

#ifndef GRAPHIC_BUFFER_SOURCE_H_

#pragma once

#define GRAPHIC_BUFFER_SOURCE_H_

#include <list>

#include <map>

#include <mutex>

#include <binder/Status.h>

#include <utils/RefBase.h>

#include <media-vndk/VndkImageReader.h>

#include <media/hardware/VideoAPI.h>

#include <media/stagefright/foundation/ABase.h>

#include <media/stagefright/foundation/AHandlerReflector.h>

#include <media/stagefright/foundation/ALooper.h>

#include <media/stagefright/bqhelper/ComponentWrapper.h>

#include <android/hardware/graphics/bufferqueue/1.0/IGraphicBufferProducer.h>

#include <android/hardware/graphics/bufferqueue/2.0/IGraphicBufferProducer.h>

namespace android {

#include <codec2/aidl/inputsurface/InputSurfaceConnection.h>

namespace aidl::android::hardware::media::c2::implementation {

struct FrameDropper;

class BufferItem;

class IGraphicBufferProducer;

class IGraphicBufferConsumer;

/*

 * This class is used to feed codecs from a Surface via BufferQueue or

 * HW producer.

/**

 * This class is used to feed codecs from ANativeWindow via AImageReader

 * for InputSurface and InputSurfaceConnection.

 *

 * Instances of the class don't run on a dedicated thread.  Instead,

 * various events trigger data movement:

 *

 *  - Availability of a new frame of data from the BufferQueue (notified

 *  - Availability of a new frame of data from the AImageReader (notified

 *    via the onFrameAvailable callback).

 *  - The return of a codec buffer.

 *  - Application signaling end-of-stream.

 * before the codec is in the "executing" state, so we need to queue

 * things up until we're ready to go.

 *

 * The GraphicBufferSource can be configure dynamically to discard frames

 * The InputSurfaceSource can be configure dynamically to discard frames

 * from the source:

 *

 * - if their timestamp is less than a start time

 * (even if it was dropped) to reencode it after an interval if no further

 * frames are sent by the producer.

 */

class GraphicBufferSource : public RefBase {

class InputSurfaceSource : public ::android::RefBase {

// TODO: remove RefBase dependency and AHanderReflector.

public:

    GraphicBufferSource();

    // creates an InputSurfaceSource.

    // init() have to be called prior to use the class.

    InputSurfaceSource();

    virtual ~InputSurfaceSource();

    virtual ~GraphicBufferSource();

    // Initialize with the default parameter. (persistent surface or init params

    // are not decided yet.)

    void init();

    // Initialize with the specified parameters. (non-persistent surface)

    void initWithParams(int32_t width, int32_t height, int32_t format,

                       int32_t maxImages, uint64_t usage);

    // We can't throw an exception if the constructor fails, so we just set

    // this and require that the caller test the value.

    status_t initCheck() const {

    c2_status_t initCheck() const {

        return mInitCheck;

    }

    // Returns the handle to the producer side of the BufferQueue.  Buffers

    // queued on this will be received by GraphicBufferSource.

    sp<IGraphicBufferProducer> getIGraphicBufferProducer() const;

    // Returns the handle to the bufferqueue HAL (V1_0) producer side of the BufferQueue.

    // Buffers queued on this will be received by GraphicBufferSource.

    sp<::android::hardware::graphics::bufferqueue::V1_0::IGraphicBufferProducer>

        getHGraphicBufferProducer_V1_0() const;

    // Returns the handle to the bufferqueue HAL producer side of the BufferQueue.

    // Buffers queued on this will be received by GraphicBufferSource.

    sp<::android::hardware::graphics::bufferqueue::V2_0::IGraphicBufferProducer>

        getHGraphicBufferProducer() const;

    /**

     * Returns the handle of ANativeWindow of the AImageReader.

     */

    ANativeWindow *getNativeWindow();

    // This is called when component transitions to running state, which means

    // we can start handing it buffers.  If we already have buffers of data

    // sitting in the BufferQueue, this will send them to the codec.

    status_t start();

    // sitting in the AImageReader, this will send them to the codec.

    c2_status_t start();

    // This is called when component transitions to stopped, indicating that

    // the codec is meant to return all buffers back to the client for them

    // to be freed. Do NOT submit any more buffers to the component.

    status_t stop();

    c2_status_t stop();

    // This is called when component transitions to released, indicating that

    // we are shutting down.

    status_t release();

    c2_status_t release();

    // A "codec buffer", i.e. a buffer that can be used to pass data into

    // the encoder, has been allocated.  (This call does not call back into

    // component.)

    status_t onInputBufferAdded(int32_t bufferId);

    // Called when encoder is no longer using the buffer.  If we have a BQ

    // buffer available, fill it with a new frame of data; otherwise, just mark

    // it as available.

    status_t onInputBufferEmptied(int32_t bufferId, int fenceFd);

    // IGraphicBufferSource interface

    // ------------------------------

    c2_status_t onInputBufferAdded(int32_t bufferId);

    // Configure the buffer source to be used with a component with the default

    // data space. (32-bit consumerUsage flag, for vendor partition

    // compatibility)

    [[deprecated("use configure() with a 64-bit consumerUsage flag instead")]]

    status_t configure(

        const sp<ComponentWrapper> &component,

        int32_t dataSpace,

        int32_t bufferCount,

        uint32_t frameWidth,

        uint32_t frameHeight,

        uint32_t consumerUsage);

    // Called when encoder is no longer using the buffer.  If we have an

    // AImageReader buffer available, fill it with a new frame of data;

    // otherwise, just mark it as available.

    c2_status_t onInputBufferEmptied(int32_t bufferId, int fenceFd);

    // Configure the buffer source to be used with a component with the default

    // data space. (64-bit consumerUsage flag)

    status_t configure(

        const sp<ComponentWrapper> &component,

    // data space.

    c2_status_t configure(

        const std::shared_ptr<c2::utils::InputSurfaceConnection> &component,

        int32_t dataSpace,

        int32_t bufferCount,

        uint32_t frameWidth,

    // timestamp is greater or equal than stopTimeUs. We need to submit an empty

    // buffer with the EOS flag set.  If we don't have a codec buffer ready,

    // we just set the mEndOfStream flag.

    status_t signalEndOfInputStream();

    c2_status_t signalEndOfInputStream();

    // If suspend is true, all incoming buffers (including those currently

    // in the BufferQueue) with timestamp larger than timeUs will be discarded

    // until the suspension is lifted. If suspend is false, all incoming buffers

    // including those currently in the BufferQueue) with timestamp larger than

    // timeUs will be processed. timeUs uses SYSTEM_TIME_MONOTONIC time base.

    status_t setSuspend(bool suspend, int64_t timeUs);

    c2_status_t setSuspend(bool suspend, int64_t timeUs);

    // Specifies the interval after which we requeue the buffer previously

    // queued to the encoder. This is useful in the case of surface flinger

    // the decoder on the remote end would be unable to decode the latest frame.

    // This API must be called before transitioning the encoder to "executing"

    // state and once this behaviour is specified it cannot be reset.

    status_t setRepeatPreviousFrameDelayUs(int64_t repeatAfterUs);

    c2_status_t setRepeatPreviousFrameDelayUs(int64_t repeatAfterUs);

    // Sets the input buffer timestamp offset.

    // When set, the sample's timestamp will be adjusted with the timeOffsetUs.

    status_t setTimeOffsetUs(int64_t timeOffsetUs);

    c2_status_t setTimeOffsetUs(int64_t timeOffsetUs);

    /*

     * Set the maximum frame rate on the source.

     *

     * When maxFps is 0, this call will fail with BAD_VALUE.

     */

    status_t setMaxFps(float maxFps);

    c2_status_t setMaxFps(float maxFps);

    // Sets the time lapse (or slow motion) parameters.

    // When set, the sample's timestamp will be modified to playback framerate,

    // and capture timestamp will be modified to capture rate.

    status_t setTimeLapseConfig(double fps, double captureFps);

    c2_status_t setTimeLapseConfig(double fps, double captureFps);

    // Sets the start time us (in system time), samples before which should

    // be dropped and not submitted to encoder

    status_t setStartTimeUs(int64_t startTimeUs);

    c2_status_t setStartTimeUs(int64_t startTimeUs);

    // Sets the stop time us (in system time), samples after which should be dropped

    // and not submitted to encoder. timeUs uses SYSTEM_TIME_MONOTONIC time base.

    status_t setStopTimeUs(int64_t stopTimeUs);

    c2_status_t setStopTimeUs(int64_t stopTimeUs);

    // Gets the stop time offset in us. This is the time offset between latest buffer

    // time and the stopTimeUs. If stop time is not set, INVALID_OPERATION will be returned.

    // If return is OK, *stopTimeOffsetUs will contain the valid offset. Otherwise,

    // *stopTimeOffsetUs will not be modified. Positive stopTimeOffsetUs means buffer time

    // larger than stopTimeUs.

    status_t getStopTimeOffsetUs(int64_t *stopTimeOffsetUs);

    c2_status_t getStopTimeOffsetUs(int64_t *stopTimeOffsetUs);

    // Sets the desired color aspects, e.g. to be used when producer does not specify a dataspace.

    status_t setColorAspects(int32_t aspectsPacked);

    c2_status_t setColorAspects(int32_t aspectsPacked);

protected:

    // BufferQueue::ConsumerListener interface, called when a new frame of

    // data is available.  If we're executing and a codec buffer is

    // available, we acquire the buffer, copy the GraphicBuffer reference

    // into the codec buffer, and call Empty[This]Buffer.  If we're not yet

    // executing or there's no codec buffer available, we just increment

    // mNumFramesAvailable and return.

    void onFrameAvailable(const BufferItem& item) ;

    // Called from AImageReader_ImageListener::onImageAvailable when a new frame

    // of  data is available. If we're executing and a codec buffer is

    // available, we acquire the buffer as an AImage, copy the AImage into the codec

    // buffer, and call Empty[This]Buffer.  If we're not yet executing or

    // there's no codec buffer available, we just increment mNumFramesAvailable

    // and return.

    void onFrameAvailable() ;

    // BufferQueue::ConsumerListener interface, called when the client has

    // released one or more GraphicBuffers.  We clear out the appropriate

    // set of mBufferSlot entries.

    void onBuffersReleased() ;

    // BufferQueue::ConsumerListener interface, called when the client has

    // changed the sideband stream. GraphicBufferSource doesn't handle sideband

    // streams so this is a no-op (and should never be called).

    void onSidebandStreamChanged() ;

    // Called from AImageReader_BufferRemovedListener::onBufferRemoved when a

    // buffer is removed. We clear an appropriate cached buffer.

    void onBufferReleased(uint64_t bid) ;

private:

    // BQ::ConsumerListener interface

    // ------------------------------

    struct ConsumerProxy;

    sp<ConsumerProxy> mConsumerProxy;

    // AImageReader listener interface

    struct ImageReaderListener;

    AImageReader_ImageListener mImageListener;

    AImageReader_BufferRemovedListener mBufferRemovedListener;

    // Lock, covers all member variables.

    mutable Mutex mMutex;

    mutable std::mutex mMutex;

    // Used to report constructor failure.

    status_t mInitCheck;

    // Used to report constructor failure regarding AImageReader creation.

    c2_status_t mInitCheck;

    // Graphic buffer reference objects

    // --------------------------------

    // These are used to keep a shared reference to GraphicBuffers and gralloc handles owned by the

    // GraphicBufferSource as well as to manage the cache slots. Separate references are owned by

    // These are used to keep a reference to AImage and gralloc handles owned by the

    // InputSurfaceSource as well as to manage the cache slots. Separate references are owned by

    // the buffer cache (controlled by the buffer queue/buffer producer) and the codec.

    // When we get a buffer from the producer (BQ) it designates them to be cached into specific

        android_dataspace_t mDataspace;

    };

    // Cached and aquired buffers

    // Cached and acquired buffers

    // --------------------------------

    typedef int slot_id;

    typedef uint64_t ahwb_id;

    typedef std::map<ahwb_id, std::shared_ptr<CachedBuffer>> BufferIdMap;

    // Maps a slot to the cached buffer in that slot

    KeyedVector<slot_id, std::shared_ptr<CachedBuffer>> mBufferSlots;

    // TODO: refactor(or remove) this not to have the buffer,

    // since it is no longer slot based

    // Maps a AHardwareBuffer id to the cached buffer

    BufferIdMap mBufferIds;

    // Queue of buffers acquired in chronological order that are not yet submitted to the codec

    List<VideoBuffer> mAvailableBuffers;

    ::std::list<VideoBuffer> mAvailableBuffers;

    // Number of buffers that have been signaled by the producer that they are available, but

    // we've been unable to acquire them due to our max acquire count

    int32_t mNumAvailableUnacquiredBuffers;

    // Number of frames acquired from consumer (debug only)

    // (as in aquireBuffer called, and release needs to be called)

    // (as in acquireBuffer called, and release needs to be called)

    int32_t mNumOutstandingAcquires;

    // Acquire a buffer from the BQ and store it in |item| if successful

    // \return OK on success, or error on failure.

    status_t acquireBuffer_l(VideoBuffer *item);

    c2_status_t acquireBuffer_l(VideoBuffer *item);

    // Called when a buffer was acquired from the producer

    void onBufferAcquired_l(const VideoBuffer &buffer);

    // marks the buffer at the slot no longer cached, and accounts for the outstanding

    // marks the buffer of the id no longer cached, and accounts for the outstanding

    // acquire count. Returns true if the slot was populated; otherwise, false.

    bool discardBufferInSlot_l(slot_id i);

    bool discardBufferInId_l(ahwb_id id);

    // marks the buffer at the slot index no longer cached, and accounts for the outstanding

    // marks the buffer at the id index no longer cached, and accounts for the outstanding

    // acquire count

    void discardBufferAtSlotIndex_l(ssize_t bsi);

    void discardBufferAtIter_l(BufferIdMap::iterator &bit);

    // release all acquired and unacquired available buffers

    // This method will return if it fails to acquire an unacquired available buffer, which will

    typedef int32_t codec_buffer_id;

    // set of codec buffer ID-s of buffers available to fill

    List<codec_buffer_id> mFreeCodecBuffers;

    std::list<codec_buffer_id> mFreeCodecBuffers;

    // maps codec buffer ID-s to buffer info submitted to the codec. Used to keep a reference for

    // the graphics buffer.

    KeyedVector<codec_buffer_id, std::shared_ptr<AcquiredBuffer>> mSubmittedCodecBuffers;

    std::map<codec_buffer_id, std::shared_ptr<AcquiredBuffer>> mSubmittedCodecBuffers;

    // Processes the next acquired frame. If there is no available codec buffer, it returns false

    // without any further action.

    // while also keeping a reference for it in mSubmittedCodecBuffers.

    // Returns UNKNOWN_ERROR if the buffer was not submitted due to buffer timestamp. Otherwise,

    // it returns any submit success or error value returned by the codec.

    status_t submitBuffer_l(const VideoBuffer &item);

    c2_status_t submitBuffer_l(const VideoBuffer &item);

    // Submits an empty buffer, with the EOS flag set if there is an available codec buffer and

    // sets mEndOfStreamSent flag. Does nothing if there is no codec buffer available.

    void onDataspaceChanged_l(android_dataspace dataspace, android_pixel_format pixelFormat);

    // Pointer back to the component that created us.  We send buffers here.

    sp<ComponentWrapper> mComponent;

    std::shared_ptr<c2::utils::InputSurfaceConnection> mComponent;

    // Set by start() / stop().

    bool mExecuting;

    int64_t mLastFrameTimestampUs;

    // Our BufferQueue interfaces. mProducer is passed to the producer through

    // getIGraphicBufferProducer, and mConsumer is used internally to retrieve

    // the buffers queued by the producer.

    sp<IGraphicBufferProducer> mProducer;

    sp<IGraphicBufferConsumer> mConsumer;

    // AImageReader creates ANativeWindow. The created ANativeWindow is passed

    // to the producer, and mImageReader is used internally to retrieve the

    // buffers queued by the producer.

    AImageReader *mImageReader;

    ANativeWindow *mImageWindow;

    // AImageReader creation parameters

    // maxImages cannot be changed after AImageReader is created.

    struct ImageReaderConfig {

        int32_t width;

        int32_t height;

        int32_t format;

        int32_t maxImages;

        uint64_t usage;

    } mImageReaderConfig;

    // The time to stop sending buffers.

    int64_t mStopTimeUs;

    // monotonically increasing.

    int64_t mLastActionTimeUs;

    // An action queue that queue up all the actions sent to GraphicBufferSource.

    // An action queue that queue up all the actions sent to InputSurfaceSource.

    // STOP action should only show up at the end of the list as all the actions

    // after a STOP action will be discarded. mActionQueue is protected by mMutex.

    List<ActionItem> mActionQueue;

    std::list<ActionItem> mActionQueue;

    ////

    friend struct AHandlerReflector<GraphicBufferSource>;

    friend struct ::android::AHandlerReflector<InputSurfaceSource>;

    enum {

        kWhatRepeatLastFrame,   ///< queue last frame for reencoding

    int64_t mSkipFramesBeforeNs;

    sp<FrameDropper> mFrameDropper;

    std::shared_ptr<FrameDropper> mFrameDropper;

    sp<ALooper> mLooper;

    sp<AHandlerReflector<GraphicBufferSource> > mReflector;

    ::android::sp<::android::ALooper> mLooper;

    ::android::sp<::android::AHandlerReflector<InputSurfaceSource> > mReflector;

    // Repeat last frame feature

    // -------------------------

    // adjustment requests.

    bool calculateCodecTimestamp_l(nsecs_t bufferTimeNs, int64_t *codecTimeUs);

    void onMessageReceived(const sp<AMessage> &msg);

    void onMessageReceived(const ::android::sp<::android::AMessage> &msg);

    DISALLOW_EVIL_CONSTRUCTORS(GraphicBufferSource);

};

    void createImageListeners();

}  // namespace android

    DISALLOW_EVIL_CONSTRUCTORS(InputSurfaceSource);

};

#endif  // GRAPHIC_BUFFER_SOURCE_H_

}  // namespace aidl::android::hardware::media::c2::implementation

/*

 * Copyright (C) 2015 The Android Open Source Project

 * Copyright (C) 2024 The Android Open Source Project

 *

 * Licensed under the Apache License, Version 2.0 (the "License");

 * you may not use this file except in compliance with the License.

 */

//#define LOG_NDEBUG 0

#define LOG_TAG "FrameDropper"

#define LOG_TAG "C2AIDL-FrameDropper"

#include <utils/Log.h>

#include <media/stagefright/bqhelper/FrameDropper.h>

#include <codec2/aidl/inputsurface/FrameDropper.h>

#include <media/stagefright/foundation/ADebug.h>

namespace android {

namespace aidl::android::hardware::media::c2::implementation {

static const int64_t kMaxJitterUs = 2000;

FrameDropper::~FrameDropper() {

}

status_t FrameDropper::setMaxFrameRate(float maxFrameRate) {

void FrameDropper::setMaxFrameRate(float maxFrameRate) {

    if (maxFrameRate < 0) {

        mMinIntervalUs = -1LL;

        return OK;

        return;

    }

    if (maxFrameRate == 0) {

        ALOGE("framerate should be positive but got %f.", maxFrameRate);

        return BAD_VALUE;

        ALOGW("framerate should be positive but got %f.", maxFrameRate);

        return;

    }

    mMinIntervalUs = (int64_t) (1000000.0f / maxFrameRate);

    return OK;

}

bool FrameDropper::shouldDrop(int64_t timeUs) {

    return false;

}

}  // namespace android

}  // namespace aidl::android::hardware::media::c2::implementation