Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit b26aebc3 authored by Leon Scroggins III's avatar Leon Scroggins III
Browse files

Implement AImageDecoder _advanceFrame and _rewind 

Bug: 160984428
Test: Iae7d274b69999c471fd5610c6ef4d148cca81bec

Disallow AImageDecoder_set* methods after the first frame, since
changing the settings would interfere with blending and caching for
kRestorePrevious frames.

Add a cache (and a state machine) for handling kRestorePrevious frames.

Follow-on to Ib93b0ced09fa3cca4a6681745406355c48158fae - support using
a matrix for unpremul + orientation (the orientation was previously
handled by a matrix internally in SkAndroidCodec).

Change-Id: I7c32ede013fa83f1fe95c35778c33278ca6fe6a3
parent 139145be

libs/hwui/hwui/ImageDecoder.cpp

+202 −36
Original line number Diff line number Diff line
@@ -17,11 +17,19 @@ 
#include "ImageDecoder.h"



#include <hwui/Bitmap.h>

#include <log/log.h>



#include <SkAndroidCodec.h>

#include <SkBitmap.h>

#include <SkBlendMode.h>

#include <SkCanvas.h>

#include <SkEncodedOrigin.h>

#include <SkFilterQuality.h>

#include <SkPaint.h>



#undef LOG_TAG

#define LOG_TAG "ImageDecoder"



using namespace android;



sk_sp<SkColorSpace> ImageDecoder::getDefaultColorSpace() const {
@@ -44,17 +52,29 @@ ImageDecoder::ImageDecoder(std::unique_ptr<SkAndroidCodec> codec, sk_sp<SkPngChu 
    , mOutColorType(mCodec->computeOutputColorType(kN32_SkColorType))

    , mUnpremultipliedRequired(false)

    , mOutColorSpace(getDefaultColorSpace())

    , mSampleSize(1)

{

    mTargetSize = swapWidthHeight() ? SkISize { mDecodeSize.height(), mDecodeSize.width() }

                                    : mDecodeSize;

    this->rewind();

}



ImageDecoder::~ImageDecoder() = default;



SkAlphaType ImageDecoder::getOutAlphaType() const {

    return opaque() ? kOpaque_SkAlphaType

                    : mUnpremultipliedRequired ? kUnpremul_SkAlphaType : kPremul_SkAlphaType;

}



static SkISize swapped(const SkISize& size) {

    return SkISize { size.height(), size.width() };

}



static bool requires_matrix_scaling(bool swapWidthHeight, const SkISize& decodeSize,

                                    const SkISize& targetSize) {

    return (swapWidthHeight && decodeSize != swapped(targetSize))

          || (!swapWidthHeight && decodeSize != targetSize);

}



bool ImageDecoder::setTargetSize(int width, int height) {

    if (width <= 0 || height <= 0) {

        return false;
@@ -78,17 +98,21 @@ bool ImageDecoder::setTargetSize(int width, int height) { 
        }

    }



    SkISize targetSize = { width, height };

    SkISize decodeSize = swapWidthHeight() ? SkISize { height, width } : targetSize;

    const bool swap = swapWidthHeight();

    const SkISize targetSize = { width, height };

    SkISize decodeSize = swap ? SkISize { height, width } : targetSize;

    int sampleSize = mCodec->computeSampleSize(&decodeSize);



    if (decodeSize != targetSize && mUnpremultipliedRequired && !opaque()) {

    if (mUnpremultipliedRequired && !opaque()) {

        // Allow using a matrix to handle orientation, but not scaling.

        if (requires_matrix_scaling(swap, decodeSize, targetSize)) {

            return false;

        }

    }



    mTargetSize = targetSize;

    mDecodeSize = decodeSize;

    mSampleSize = sampleSize;

    mOptions.fSampleSize = sampleSize;

    return true;

}
@@ -137,9 +161,11 @@ bool ImageDecoder::setOutColorType(SkColorType colorType) { 
}



bool ImageDecoder::setUnpremultipliedRequired(bool required) {

    if (required && !opaque() && mDecodeSize != mTargetSize) {

    if (required && !opaque()) {

        if (requires_matrix_scaling(swapWidthHeight(), mDecodeSize, mTargetSize)) {

            return false;

        }

    }

    mUnpremultipliedRequired = required;

    return true;

}
@@ -176,26 +202,173 @@ int ImageDecoder::height() const { 
}



bool ImageDecoder::opaque() const {

    return mCodec->getInfo().alphaType() == kOpaque_SkAlphaType;

    return mCurrentFrameIsOpaque;

}



bool ImageDecoder::gray() const {

    return mCodec->getInfo().colorType() == kGray_8_SkColorType;

}



bool ImageDecoder::isAnimated() {

    return mCodec->codec()->getFrameCount() > 1;

}



int ImageDecoder::currentFrame() const {

    return mOptions.fFrameIndex;

}



bool ImageDecoder::rewind() {

    mOptions.fFrameIndex = 0;

    mOptions.fPriorFrame = SkCodec::kNoFrame;

    mCurrentFrameIsIndependent = true;

    mCurrentFrameIsOpaque = mCodec->getInfo().isOpaque();

    mRestoreState = RestoreState::kDoNothing;

    mRestoreFrame = nullptr;



    // TODO: Rewind the input now instead of in the next call to decode, and

    // plumb through whether rewind succeeded.

    return true;

}



bool ImageDecoder::advanceFrame() {

    const int frameIndex = ++mOptions.fFrameIndex;

    const int frameCount = mCodec->codec()->getFrameCount();

    if (frameIndex >= frameCount) {

        // Prevent overflow from repeated calls to advanceFrame.

        mOptions.fFrameIndex = frameCount;

        return false;

    }



    SkCodec::FrameInfo frameInfo;

    if (!mCodec->codec()->getFrameInfo(frameIndex, &frameInfo)

            || !frameInfo.fFullyReceived) {

        // Mark the decoder as finished, requiring a rewind.

        mOptions.fFrameIndex = frameCount;

        return false;

    }



    mCurrentFrameIsIndependent = frameInfo.fRequiredFrame == SkCodec::kNoFrame;

    mCurrentFrameIsOpaque = frameInfo.fAlphaType == kOpaque_SkAlphaType;



    if (frameInfo.fDisposalMethod == SkCodecAnimation::DisposalMethod::kRestorePrevious) {

        switch (mRestoreState) {

            case RestoreState::kDoNothing:

            case RestoreState::kNeedsRestore:

                mRestoreState = RestoreState::kFirstRPFrame;

                break;

            case RestoreState::kFirstRPFrame:

                mRestoreState = RestoreState::kRPFrame;

                break;

            case RestoreState::kRPFrame:

                // Unchanged.

                break;

        }

    } else { // New frame is not restore previous

        switch (mRestoreState) {

            case RestoreState::kFirstRPFrame:

            case RestoreState::kRPFrame:

                mRestoreState = RestoreState::kNeedsRestore;

                break;

            case RestoreState::kNeedsRestore:

                mRestoreState = RestoreState::kDoNothing;

                mRestoreFrame = nullptr;

                [[fallthrough]];

            case RestoreState::kDoNothing:

                mOptions.fPriorFrame = frameIndex - 1;

                break;

        }

    }



    return true;

}



bool ImageDecoder::finished() const {

    return mOptions.fFrameIndex >= mCodec->codec()->getFrameCount();

}



SkCodec::Result ImageDecoder::decode(void* pixels, size_t rowBytes) {

    // This was checked inside setTargetSize, but it's possible the first frame

    // was opaque, so that method succeeded, but after calling advanceFrame, the

    // current frame is not opaque.

    if (mUnpremultipliedRequired && !opaque()) {

        // Allow using a matrix to handle orientation, but not scaling.

        if (requires_matrix_scaling(swapWidthHeight(), mDecodeSize, mTargetSize)) {

            return SkCodec::kInvalidScale;

        }

    }



    void* decodePixels = pixels;

    size_t decodeRowBytes = rowBytes;

    auto decodeInfo = SkImageInfo::Make(mDecodeSize, mOutColorType, getOutAlphaType(),

    const auto decodeInfo = SkImageInfo::Make(mDecodeSize, mOutColorType, getOutAlphaType(),

                                              getOutputColorSpace());

    const auto outputInfo = getOutputInfo();

    switch (mRestoreState) {

        case RestoreState::kFirstRPFrame:{

            // This frame is marked kRestorePrevious. The prior frame should be in

            // |pixels|, and it is what we'll restore after each consecutive

            // kRestorePrevious frame. Cache it now.

            if (!(mRestoreFrame = Bitmap::allocateHeapBitmap(outputInfo))) {

                return SkCodec::kInternalError;

            }



            const uint8_t* srcRow = static_cast<uint8_t*>(pixels);

                  uint8_t* dstRow = static_cast<uint8_t*>(mRestoreFrame->pixels());

            for (int y = 0; y < outputInfo.height(); y++) {

                memcpy(dstRow, srcRow, outputInfo.minRowBytes());

                srcRow += rowBytes;

                dstRow += mRestoreFrame->rowBytes();

            }

            break;

        }

        case RestoreState::kRPFrame:

        case RestoreState::kNeedsRestore:

            // Restore the cached frame. It's possible that the client skipped decoding a frame, so

            // we never cached it.

            if (mRestoreFrame) {

                const uint8_t* srcRow = static_cast<uint8_t*>(mRestoreFrame->pixels());

                      uint8_t* dstRow = static_cast<uint8_t*>(pixels);

                for (int y = 0; y < outputInfo.height(); y++) {

                    memcpy(dstRow, srcRow, outputInfo.minRowBytes());

                    srcRow += mRestoreFrame->rowBytes();

                    dstRow += rowBytes;

                }

            }

            break;

        case RestoreState::kDoNothing:

            break;

    }



    // Used if we need a temporary before scaling or subsetting.

    // FIXME: Use scanline decoding on only a couple lines to save memory. b/70709380.

    SkBitmap tmp;

    const bool scale = mDecodeSize != mTargetSize;

    const auto origin = mCodec->codec()->getOrigin();

    const bool handleOrigin = origin != kDefault_SkEncodedOrigin;

    SkMatrix outputMatrix;

    if (scale || handleOrigin || mCropRect) {

        if (!tmp.setInfo(decodeInfo)) {

        if (mCropRect) {

            outputMatrix.setTranslate(-mCropRect->fLeft, -mCropRect->fTop);

        }



        int targetWidth  = mTargetSize.width();

        int targetHeight = mTargetSize.height();

        if (handleOrigin) {

            outputMatrix.preConcat(SkEncodedOriginToMatrix(origin, targetWidth, targetHeight));

            if (SkEncodedOriginSwapsWidthHeight(origin)) {

                std::swap(targetWidth, targetHeight);

            }

        }

        if (scale) {

            float scaleX = (float) targetWidth  / mDecodeSize.width();

            float scaleY = (float) targetHeight / mDecodeSize.height();

            outputMatrix.preScale(scaleX, scaleY);

        }

        // It's possible that this portion *does* have alpha, even if the

        // composed frame does not. In that case, the SkBitmap needs to have

        // alpha so it blends properly.

        if (!tmp.setInfo(decodeInfo.makeAlphaType(mUnpremultipliedRequired ? kUnpremul_SkAlphaType

                                                                           : kPremul_SkAlphaType)))

        {

            return SkCodec::kInternalError;

        }

        if (!Bitmap::allocateHeapBitmap(&tmp)) {
@@ -203,15 +376,28 @@ SkCodec::Result ImageDecoder::decode(void* pixels, size_t rowBytes) { 
        }

        decodePixels = tmp.getPixels();

        decodeRowBytes = tmp.rowBytes();



        if (!mCurrentFrameIsIndependent) {

            SkMatrix inverse;

            if (outputMatrix.invert(&inverse)) {

                SkCanvas canvas(tmp, SkCanvas::ColorBehavior::kLegacy);

                canvas.setMatrix(inverse);

                SkPaint paint;

                paint.setFilterQuality(kLow_SkFilterQuality); // bilinear

                SkBitmap priorFrame;

                priorFrame.installPixels(outputInfo, pixels, rowBytes);

                canvas.drawBitmap(priorFrame, 0, 0, &paint);

            } else {

                ALOGE("Failed to invert matrix!");

            }

        }

    }



    SkAndroidCodec::AndroidOptions options;

    options.fSampleSize = mSampleSize;

    auto result = mCodec->getAndroidPixels(decodeInfo, decodePixels, decodeRowBytes, &options);

    auto result = mCodec->getAndroidPixels(decodeInfo, decodePixels, decodeRowBytes, &mOptions);



    if (scale || handleOrigin || mCropRect) {

        SkBitmap scaledBm;

        if (!scaledBm.installPixels(getOutputInfo(), pixels, rowBytes)) {

        if (!scaledBm.installPixels(outputInfo, pixels, rowBytes)) {

            return SkCodec::kInternalError;

        }
@@ -220,26 +406,6 @@ SkCodec::Result ImageDecoder::decode(void* pixels, size_t rowBytes) { 
        paint.setFilterQuality(kLow_SkFilterQuality);  // bilinear filtering



        SkCanvas canvas(scaledBm, SkCanvas::ColorBehavior::kLegacy);

        SkMatrix outputMatrix;

        if (mCropRect) {

            outputMatrix.setTranslate(-mCropRect->fLeft, -mCropRect->fTop);

        }



        int targetWidth  = mTargetSize.width();

        int targetHeight = mTargetSize.height();

        if (handleOrigin) {

            outputMatrix.preConcat(SkEncodedOriginToMatrix(origin, targetWidth, targetHeight));

            if (SkEncodedOriginSwapsWidthHeight(origin)) {

                std::swap(targetWidth, targetHeight);

            }

        }



        if (scale) {

            float scaleX = (float) targetWidth  / mDecodeSize.width();

            float scaleY = (float) targetHeight / mDecodeSize.height();

            outputMatrix.preScale(scaleX, scaleY);

        }



        canvas.setMatrix(outputMatrix);

        canvas.drawBitmap(tmp, 0.0f, 0.0f, &paint);

    }

libs/hwui/hwui/ImageDecoder.h

+45 −5
Original line number Diff line number Diff line
@@ -15,6 +15,7 @@ 
 */

#pragma once



#include <SkAndroidCodec.h>

#include <SkCodec.h>

#include <SkImageInfo.h>

#include <SkPngChunkReader.h>
@@ -24,17 +25,18 @@ 


#include <optional>



class SkAndroidCodec;



namespace android {



class ANDROID_API ImageDecoder {

class Bitmap;



class ANDROID_API ImageDecoder final {

public:

    std::unique_ptr<SkAndroidCodec> mCodec;

    sk_sp<SkPngChunkReader> mPeeker;



    ImageDecoder(std::unique_ptr<SkAndroidCodec> codec,

                 sk_sp<SkPngChunkReader> peeker = nullptr);

    ~ImageDecoder();



    bool setTargetSize(int width, int height);

    bool setCropRect(const SkIRect*);
@@ -46,25 +48,63 @@ public: 
    sk_sp<SkColorSpace> getDefaultColorSpace() const;

    void setOutColorSpace(sk_sp<SkColorSpace> cs);



    // The size is the final size after scaling and cropping.

    // The size is the final size after scaling, adjusting for the origin, and

    // cropping.

    SkImageInfo getOutputInfo() const;



    int width() const;

    int height() const;



    // True if the current frame is opaque.

    bool opaque() const;



    bool gray() const;



    SkCodec::Result decode(void* pixels, size_t rowBytes);



    // Return true if the decoder has advanced beyond all frames.

    bool finished() const;



    bool advanceFrame();

    bool rewind();



    bool isAnimated();

    int currentFrame() const;



private:

    // State machine for keeping track of how to handle RestorePrevious (RP)

    // frames in decode().

    enum class RestoreState {

        // Neither this frame nor the prior is RP, so there is no need to cache

        // or restore.

        kDoNothing,



        // This is the first in a sequence of one or more RP frames. decode()

        // needs to cache the provided pixels.

        kFirstRPFrame,



        // This is the second (or later) in a sequence of multiple RP frames.

        // decode() needs to restore the cached frame that preceded the first RP

        // frame in the sequence.

        kRPFrame,



        // This is the first non-RP frame after a sequence of one or more RP

        // frames. decode() still needs to restore the cached frame. Separate

        // from kRPFrame because if the following frame is RP the state will

        // change to kFirstRPFrame.

        kNeedsRestore,

    };



    SkISize mTargetSize;

    SkISize mDecodeSize;

    SkColorType mOutColorType;

    bool mUnpremultipliedRequired;

    sk_sp<SkColorSpace> mOutColorSpace;

    int mSampleSize;

    SkAndroidCodec::AndroidOptions mOptions;

    bool mCurrentFrameIsIndependent;

    bool mCurrentFrameIsOpaque;

    RestoreState mRestoreState;

    sk_sp<Bitmap> mRestoreFrame;

    std::optional<SkIRect> mCropRect;



    ImageDecoder(const ImageDecoder&) = delete;

native/graphics/jni/imagedecoder.cpp

+65 −5
Original line number Diff line number Diff line
@@ -173,7 +173,13 @@ int AImageDecoder_setAndroidBitmapFormat(AImageDecoder* decoder, int32_t format) 
            || format > ANDROID_BITMAP_FORMAT_RGBA_F16) {

        return ANDROID_IMAGE_DECODER_BAD_PARAMETER;

    }

    return toDecoder(decoder)->setOutColorType(getColorType((AndroidBitmapFormat) format))



    auto* imageDecoder = toDecoder(decoder);

    if (imageDecoder->currentFrame() != 0) {

        return ANDROID_IMAGE_DECODER_INVALID_STATE;

    }



    return imageDecoder->setOutColorType(getColorType((AndroidBitmapFormat) format))

            ? ANDROID_IMAGE_DECODER_SUCCESS : ANDROID_IMAGE_DECODER_INVALID_CONVERSION;

}
@@ -185,6 +191,10 @@ int AImageDecoder_setDataSpace(AImageDecoder* decoder, int32_t dataspace) { 
    }



    ImageDecoder* imageDecoder = toDecoder(decoder);

    if (imageDecoder->currentFrame() != 0) {

        return ANDROID_IMAGE_DECODER_INVALID_STATE;

    }



    imageDecoder->setOutColorSpace(std::move(cs));

    return ANDROID_IMAGE_DECODER_SUCCESS;

}
@@ -279,7 +289,12 @@ int AImageDecoder_setUnpremultipliedRequired(AImageDecoder* decoder, bool requir 
        return ANDROID_IMAGE_DECODER_BAD_PARAMETER;

    }



    return toDecoder(decoder)->setUnpremultipliedRequired(required)

    auto* imageDecoder = toDecoder(decoder);

    if (imageDecoder->currentFrame() != 0) {

        return ANDROID_IMAGE_DECODER_INVALID_STATE;

    }



    return imageDecoder->setUnpremultipliedRequired(required)

            ? ANDROID_IMAGE_DECODER_SUCCESS : ANDROID_IMAGE_DECODER_INVALID_CONVERSION;

}
@@ -288,7 +303,12 @@ int AImageDecoder_setTargetSize(AImageDecoder* decoder, int32_t width, int32_t h 
        return ANDROID_IMAGE_DECODER_BAD_PARAMETER;

    }



    return toDecoder(decoder)->setTargetSize(width, height)

    auto* imageDecoder = toDecoder(decoder);

    if (imageDecoder->currentFrame() != 0) {

        return ANDROID_IMAGE_DECODER_INVALID_STATE;

    }



    return imageDecoder->setTargetSize(width, height)

            ? ANDROID_IMAGE_DECODER_SUCCESS : ANDROID_IMAGE_DECODER_INVALID_SCALE;

}
@@ -309,10 +329,15 @@ int AImageDecoder_setCrop(AImageDecoder* decoder, ARect crop) { 
        return ANDROID_IMAGE_DECODER_BAD_PARAMETER;

    }



    auto* imageDecoder = toDecoder(decoder);

    if (imageDecoder->currentFrame() != 0) {

        return ANDROID_IMAGE_DECODER_INVALID_STATE;

    }



    SkIRect cropIRect;

    cropIRect.setLTRB(crop.left, crop.top, crop.right, crop.bottom);

    SkIRect* cropPtr = cropIRect == SkIRect::MakeEmpty() ? nullptr : &cropIRect;

    return toDecoder(decoder)->setCropRect(cropPtr)

    return imageDecoder->setCropRect(cropPtr)

            ? ANDROID_IMAGE_DECODER_SUCCESS : ANDROID_IMAGE_DECODER_BAD_PARAMETER;

}
@@ -341,6 +366,10 @@ int AImageDecoder_decodeImage(AImageDecoder* decoder, 
        return ANDROID_IMAGE_DECODER_BAD_PARAMETER;

    }



    if (imageDecoder->finished()) {

        return ANDROID_IMAGE_DECODER_FINISHED;

    }



    return ResultToErrorCode(imageDecoder->decode(pixels, stride));

}
@@ -352,7 +381,7 @@ bool AImageDecoder_isAnimated(AImageDecoder* decoder) { 
    if (!decoder) return false;



    ImageDecoder* imageDecoder = toDecoder(decoder);

    return imageDecoder->mCodec->codec()->getFrameCount() > 1;

    return imageDecoder->isAnimated();

}



int32_t AImageDecoder_getRepeatCount(AImageDecoder* decoder) {
@@ -369,3 +398,34 @@ int32_t AImageDecoder_getRepeatCount(AImageDecoder* decoder) { 
    }

    return count;

}



int AImageDecoder_advanceFrame(AImageDecoder* decoder) {

    if (!decoder) return ANDROID_IMAGE_DECODER_BAD_PARAMETER;



    ImageDecoder* imageDecoder = toDecoder(decoder);

    if (!imageDecoder->isAnimated()) {

        return ANDROID_IMAGE_DECODER_BAD_PARAMETER;

    }



    if (imageDecoder->advanceFrame()) {

        return ANDROID_IMAGE_DECODER_SUCCESS;

    }



    if (imageDecoder->finished()) {

        return ANDROID_IMAGE_DECODER_FINISHED;

    }



    return ANDROID_IMAGE_DECODER_INCOMPLETE;

}



int AImageDecoder_rewind(AImageDecoder* decoder) {

    if (!decoder) return ANDROID_IMAGE_DECODER_BAD_PARAMETER;



    ImageDecoder* imageDecoder = toDecoder(decoder);

    if (!imageDecoder->isAnimated()) {

        return ANDROID_IMAGE_DECODER_BAD_PARAMETER;

    }



    return imageDecoder->rewind() ? ANDROID_IMAGE_DECODER_SUCCESS

                                  : ANDROID_IMAGE_DECODER_SEEK_ERROR;

}

native/graphics/jni/libjnigraphics.map.txt

+2 −0
Original line number Diff line number Diff line
@@ -15,6 +15,8 @@ LIBJNIGRAPHICS { 
    AImageDecoder_setCrop; # introduced=30

    AImageDecoder_isAnimated; # introduced=31

    AImageDecoder_getRepeatCount; # introduced=31

    AImageDecoder_advanceFrame; # introduced=31

    AImageDecoder_rewind; # introduced=31

    AImageDecoderHeaderInfo_getWidth; # introduced=30

    AImageDecoderHeaderInfo_getHeight; # introduced=30

    AImageDecoderHeaderInfo_getMimeType; # introduced=30