Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 4faf072e authored by Xusong Wang's avatar Xusong Wang
Browse files

Add VTS test for dynamic output shape. 

Test dynamic output shape with generated models when
- Dimensions of output operands are fully specified
- Dimensions of output operands are unspecified with sufficient buffer
- Dimensions of output operands are unspecified with insufficient buffer

Test: VTS on 1.2 sample driver
Change-Id: I4d26395ce443687ccbd47445b36e3356d70035cc
Merged-In: I4d26395ce443687ccbd47445b36e3356d70035cc
(cherry picked from commit 929fd21e)
parent 2739b2ef

neuralnetworks/1.0/vts/functional/GeneratedTestHarness.cpp

+109 −33
Original line number Diff line number Diff line
@@ -120,12 +120,14 @@ static std::unique_ptr<::android::nn::ExecutionBurstController> CreateBurst( 
    return ::android::nn::createExecutionBurstController(preparedModel, /*blocking=*/true);

}

enum class Executor { ASYNC, SYNC, BURST };

enum class OutputType { FULLY_SPECIFIED, UNSPECIFIED, INSUFFICIENT };

const float kDefaultAtol = 1e-5f;

const float kDefaultRtol = 1e-5f;

template <typename T_IPreparedModel>

void EvaluatePreparedModel(sp<T_IPreparedModel>& preparedModel, std::function<bool(int)> is_ignored,

        const std::vector<MixedTypedExample>& examples, bool hasRelaxedFloat32Model, float fpAtol,

        float fpRtol, Executor executor, MeasureTiming measure, bool testDynamicOutputShape) {

                           const std::vector<MixedTypedExample>& examples,

                           bool hasRelaxedFloat32Model, float fpAtol, float fpRtol,

                           Executor executor, MeasureTiming measure, OutputType outputType) {

    const uint32_t INPUT = 0;

    const uint32_t OUTPUT = 1;
@@ -173,8 +175,20 @@ void EvaluatePreparedModel(sp<T_IPreparedModel>& preparedModel, std::function<bo 


        // Go through all outputs, initialize RequestArgument descriptors

        resize_accordingly(golden, test);

        for_all(golden, [&outputs_info, &outputSize](int index, auto, auto s) {

        bool sizeLargerThanOne = true;

        for_all(golden, [&outputs_info, &outputSize, &outputType, &sizeLargerThanOne](

                                int index, auto, auto s) {

            if (outputs_info.size() <= static_cast<size_t>(index)) outputs_info.resize(index + 1);

            if (index == 0) {

                // On OutputType::INSUFFICIENT, set the output operand with index 0 with

                // buffer size one byte less than needed.

                if (outputType == OutputType::INSUFFICIENT) {

                    if (s > 1)

                        s -= 1;

                    else

                        sizeLargerThanOne = false;

                }

            }

            RequestArgument arg = {

                .location = {.poolIndex = OUTPUT, .offset = 0, .length = static_cast<uint32_t>(s)},

                .dimensions = {},
@@ -182,6 +196,9 @@ void EvaluatePreparedModel(sp<T_IPreparedModel>& preparedModel, std::function<bo 
            outputs_info[index] = arg;

            outputSize += s;

        });

        // If output0 does not have size larger than one byte,

        // we can not provide an insufficient buffer

        if (!sizeLargerThanOne && outputType == OutputType::INSUFFICIENT) return;

        // Compute offset for outputs 1 and so on

        {

            size_t offset = 0;
@@ -276,15 +293,15 @@ void EvaluatePreparedModel(sp<T_IPreparedModel>& preparedModel, std::function<bo 
            }

        }



        if (testDynamicOutputShape && executionStatus != ErrorStatus::NONE) {

        if (outputType != OutputType::FULLY_SPECIFIED &&

            executionStatus == ErrorStatus::GENERAL_FAILURE) {

            LOG(INFO) << "NN VTS: Early termination of test because vendor service cannot "

                         "execute model that it does not support.";

            std::cout << "[          ]   Early termination of test because vendor service cannot "

                         "execute model that it does not support."

                      << std::endl;

            return;

            GTEST_SKIP();

        }

        ASSERT_EQ(ErrorStatus::NONE, executionStatus);

        if (measure == MeasureTiming::NO) {

            EXPECT_EQ(UINT64_MAX, timing.timeOnDevice);

            EXPECT_EQ(UINT64_MAX, timing.timeInDriver);
@@ -294,9 +311,28 @@ void EvaluatePreparedModel(sp<T_IPreparedModel>& preparedModel, std::function<bo 
            }

        }



        switch (outputType) {

            case OutputType::FULLY_SPECIFIED:

                // If the model output operands are fully specified, outputShapes must be either

                // either empty, or have the same number of elements as the number of outputs.

                ASSERT_EQ(ErrorStatus::NONE, executionStatus);

                ASSERT_TRUE(outputShapes.size() == 0 ||

                            outputShapes.size() == test.operandDimensions.size());

                break;

            case OutputType::UNSPECIFIED:

                // If the model output operands are not fully specified, outputShapes must have

                // the same number of elements as the number of outputs.

                ASSERT_EQ(ErrorStatus::NONE, executionStatus);

                ASSERT_EQ(outputShapes.size(), test.operandDimensions.size());

                break;

            case OutputType::INSUFFICIENT:

                ASSERT_EQ(ErrorStatus::OUTPUT_INSUFFICIENT_SIZE, executionStatus);

                ASSERT_EQ(outputShapes.size(), test.operandDimensions.size());

                ASSERT_FALSE(outputShapes[0].isSufficient);

                return;

        }

        // Go through all outputs, overwrite output dimensions with returned output shapes

        if (testDynamicOutputShape) {

            ASSERT_NE(outputShapes.size(), 0);

        if (outputShapes.size() > 0) {

            for_each<uint32_t>(test.operandDimensions,

                               [&outputShapes](int idx, std::vector<uint32_t>& dim) {

                                   dim = outputShapes[idx].dimensions;
@@ -321,10 +357,11 @@ void EvaluatePreparedModel(sp<T_IPreparedModel>& preparedModel, std::function<bo 
}

template <typename T_IPreparedModel>

void EvaluatePreparedModel(sp<T_IPreparedModel>& preparedModel, std::function<bool(int)> is_ignored,

        const std::vector<MixedTypedExample>& examples, bool hasRelaxedFloat32Model,

        Executor executor, MeasureTiming measure, bool testDynamicOutputShape) {

                           const std::vector<MixedTypedExample>& examples,

                           bool hasRelaxedFloat32Model, Executor executor, MeasureTiming measure,

                           OutputType outputType) {

    EvaluatePreparedModel(preparedModel, is_ignored, examples, hasRelaxedFloat32Model, kDefaultAtol,

            kDefaultRtol, executor, measure, testDynamicOutputShape);

                          kDefaultRtol, executor, measure, outputType);

}



static void getPreparedModel(sp<PreparedModelCallback> callback,
@@ -380,8 +417,8 @@ void Execute(const sp<V1_0::IDevice>& device, std::function<V1_0::Model(void)> c 


    float fpAtol = 1e-5f, fpRtol = 5.0f * 1.1920928955078125e-7f;

    EvaluatePreparedModel(preparedModel, is_ignored, examples,

            /*hasRelaxedFloat32Model=*/false, fpAtol, fpRtol, Executor::ASYNC, MeasureTiming::NO,

            /*testDynamicOutputShape=*/false);

                          /*hasRelaxedFloat32Model=*/false, fpAtol, fpRtol, Executor::ASYNC,

                          MeasureTiming::NO, OutputType::FULLY_SPECIFIED);

}



void Execute(const sp<V1_1::IDevice>& device, std::function<V1_1::Model(void)> create_model,
@@ -428,7 +465,7 @@ void Execute(const sp<V1_1::IDevice>& device, std::function<V1_1::Model(void)> c 


    EvaluatePreparedModel(preparedModel, is_ignored, examples,

                          model.relaxComputationFloat32toFloat16, 1e-5f, 1e-5f, Executor::ASYNC,

            MeasureTiming::NO, /*testDynamicOutputShape=*/false);

                          MeasureTiming::NO, OutputType::FULLY_SPECIFIED);

}



// TODO: Reduce code duplication.
@@ -475,24 +512,63 @@ void Execute(const sp<V1_2::IDevice>& device, std::function<V1_2::Model(void)> c 
    EXPECT_EQ(ErrorStatus::NONE, prepareReturnStatus);

    ASSERT_NE(nullptr, preparedModel.get());



    if (testDynamicOutputShape) {

        EvaluatePreparedModel(preparedModel, is_ignored, examples,

                              model.relaxComputationFloat32toFloat16, Executor::ASYNC,

                              MeasureTiming::NO, OutputType::UNSPECIFIED);

        EvaluatePreparedModel(preparedModel, is_ignored, examples,

            model.relaxComputationFloat32toFloat16, Executor::ASYNC, MeasureTiming::NO,

            testDynamicOutputShape);

                              model.relaxComputationFloat32toFloat16, Executor::SYNC,

                              MeasureTiming::NO, OutputType::UNSPECIFIED);

        EvaluatePreparedModel(preparedModel, is_ignored, examples,

            model.relaxComputationFloat32toFloat16, Executor::SYNC, MeasureTiming::NO,

            testDynamicOutputShape);

                              model.relaxComputationFloat32toFloat16, Executor::BURST,

                              MeasureTiming::NO, OutputType::UNSPECIFIED);

        EvaluatePreparedModel(preparedModel, is_ignored, examples,

            model.relaxComputationFloat32toFloat16, Executor::BURST, MeasureTiming::NO,

            testDynamicOutputShape);

                              model.relaxComputationFloat32toFloat16, Executor::ASYNC,

                              MeasureTiming::YES, OutputType::UNSPECIFIED);

        EvaluatePreparedModel(preparedModel, is_ignored, examples,

            model.relaxComputationFloat32toFloat16, Executor::ASYNC, MeasureTiming::YES,

            testDynamicOutputShape);

                              model.relaxComputationFloat32toFloat16, Executor::SYNC,

                              MeasureTiming::YES, OutputType::UNSPECIFIED);

        EvaluatePreparedModel(preparedModel, is_ignored, examples,

            model.relaxComputationFloat32toFloat16, Executor::SYNC, MeasureTiming::YES,

            testDynamicOutputShape);

                              model.relaxComputationFloat32toFloat16, Executor::BURST,

                              MeasureTiming::YES, OutputType::UNSPECIFIED);

        EvaluatePreparedModel(preparedModel, is_ignored, examples,

            model.relaxComputationFloat32toFloat16, Executor::BURST, MeasureTiming::YES,

            testDynamicOutputShape);

                              model.relaxComputationFloat32toFloat16, Executor::ASYNC,

                              MeasureTiming::NO, OutputType::INSUFFICIENT);

        EvaluatePreparedModel(preparedModel, is_ignored, examples,

                              model.relaxComputationFloat32toFloat16, Executor::SYNC,

                              MeasureTiming::NO, OutputType::INSUFFICIENT);

        EvaluatePreparedModel(preparedModel, is_ignored, examples,

                              model.relaxComputationFloat32toFloat16, Executor::BURST,

                              MeasureTiming::NO, OutputType::INSUFFICIENT);

        EvaluatePreparedModel(preparedModel, is_ignored, examples,

                              model.relaxComputationFloat32toFloat16, Executor::ASYNC,

                              MeasureTiming::YES, OutputType::INSUFFICIENT);

        EvaluatePreparedModel(preparedModel, is_ignored, examples,

                              model.relaxComputationFloat32toFloat16, Executor::SYNC,

                              MeasureTiming::YES, OutputType::INSUFFICIENT);

        EvaluatePreparedModel(preparedModel, is_ignored, examples,

                              model.relaxComputationFloat32toFloat16, Executor::BURST,

                              MeasureTiming::YES, OutputType::INSUFFICIENT);

    } else {

        EvaluatePreparedModel(preparedModel, is_ignored, examples,

                              model.relaxComputationFloat32toFloat16, Executor::ASYNC,

                              MeasureTiming::NO, OutputType::FULLY_SPECIFIED);

        EvaluatePreparedModel(preparedModel, is_ignored, examples,

                              model.relaxComputationFloat32toFloat16, Executor::SYNC,

                              MeasureTiming::NO, OutputType::FULLY_SPECIFIED);

        EvaluatePreparedModel(preparedModel, is_ignored, examples,

                              model.relaxComputationFloat32toFloat16, Executor::BURST,

                              MeasureTiming::NO, OutputType::FULLY_SPECIFIED);

        EvaluatePreparedModel(preparedModel, is_ignored, examples,

                              model.relaxComputationFloat32toFloat16, Executor::ASYNC,

                              MeasureTiming::YES, OutputType::FULLY_SPECIFIED);

        EvaluatePreparedModel(preparedModel, is_ignored, examples,

                              model.relaxComputationFloat32toFloat16, Executor::SYNC,

                              MeasureTiming::YES, OutputType::FULLY_SPECIFIED);

        EvaluatePreparedModel(preparedModel, is_ignored, examples,

                              model.relaxComputationFloat32toFloat16, Executor::BURST,

                              MeasureTiming::YES, OutputType::FULLY_SPECIFIED);

    }

}



}  // namespace generated_tests