Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 13809338 authored by Xusong Wang's avatar Xusong Wang Committed by android-build-merger
Browse files

Add VTS tests for NNAPI compilation caching. 

am: 96e68dc8

Change-Id: If2c8d1fd88ab92c8ec6dfe8fb5ab73f59d9b4e01
parents 0fdf3413 96e68dc8

neuralnetworks/1.0/vts/functional/GeneratedTestHarness.cpp

+61 −67
Original line number Diff line number Diff line
@@ -14,6 +14,7 @@ 
 * limitations under the License.

 */



#include "GeneratedTestHarness.h"

#include "Callbacks.h"

#include "ExecutionBurstController.h"

#include "TestHarness.h"
@@ -364,6 +365,51 @@ void EvaluatePreparedModel(sp<T_IPreparedModel>& preparedModel, std::function<bo 
                          kDefaultRtol, executor, measure, outputType);

}



void EvaluatePreparedModel(sp<V1_2::IPreparedModel>& preparedModel,

                           std::function<bool(int)> is_ignored,

                           const std::vector<MixedTypedExample>& examples,

                           bool hasRelaxedFloat32Model, bool testDynamicOutputShape) {

    if (testDynamicOutputShape) {

        EvaluatePreparedModel(preparedModel, is_ignored, examples, hasRelaxedFloat32Model,

                              Executor::ASYNC, MeasureTiming::NO, OutputType::UNSPECIFIED);

        EvaluatePreparedModel(preparedModel, is_ignored, examples, hasRelaxedFloat32Model,

                              Executor::SYNC, MeasureTiming::NO, OutputType::UNSPECIFIED);

        EvaluatePreparedModel(preparedModel, is_ignored, examples, hasRelaxedFloat32Model,

                              Executor::BURST, MeasureTiming::NO, OutputType::UNSPECIFIED);

        EvaluatePreparedModel(preparedModel, is_ignored, examples, hasRelaxedFloat32Model,

                              Executor::ASYNC, MeasureTiming::YES, OutputType::UNSPECIFIED);

        EvaluatePreparedModel(preparedModel, is_ignored, examples, hasRelaxedFloat32Model,

                              Executor::SYNC, MeasureTiming::YES, OutputType::UNSPECIFIED);

        EvaluatePreparedModel(preparedModel, is_ignored, examples, hasRelaxedFloat32Model,

                              Executor::BURST, MeasureTiming::YES, OutputType::UNSPECIFIED);

        EvaluatePreparedModel(preparedModel, is_ignored, examples, hasRelaxedFloat32Model,

                              Executor::ASYNC, MeasureTiming::NO, OutputType::INSUFFICIENT);

        EvaluatePreparedModel(preparedModel, is_ignored, examples, hasRelaxedFloat32Model,

                              Executor::SYNC, MeasureTiming::NO, OutputType::INSUFFICIENT);

        EvaluatePreparedModel(preparedModel, is_ignored, examples, hasRelaxedFloat32Model,

                              Executor::BURST, MeasureTiming::NO, OutputType::INSUFFICIENT);

        EvaluatePreparedModel(preparedModel, is_ignored, examples, hasRelaxedFloat32Model,

                              Executor::ASYNC, MeasureTiming::YES, OutputType::INSUFFICIENT);

        EvaluatePreparedModel(preparedModel, is_ignored, examples, hasRelaxedFloat32Model,

                              Executor::SYNC, MeasureTiming::YES, OutputType::INSUFFICIENT);

        EvaluatePreparedModel(preparedModel, is_ignored, examples, hasRelaxedFloat32Model,

                              Executor::BURST, MeasureTiming::YES, OutputType::INSUFFICIENT);

    } else {

        EvaluatePreparedModel(preparedModel, is_ignored, examples, hasRelaxedFloat32Model,

                              Executor::ASYNC, MeasureTiming::NO, OutputType::FULLY_SPECIFIED);

        EvaluatePreparedModel(preparedModel, is_ignored, examples, hasRelaxedFloat32Model,

                              Executor::SYNC, MeasureTiming::NO, OutputType::FULLY_SPECIFIED);

        EvaluatePreparedModel(preparedModel, is_ignored, examples, hasRelaxedFloat32Model,

                              Executor::BURST, MeasureTiming::NO, OutputType::FULLY_SPECIFIED);

        EvaluatePreparedModel(preparedModel, is_ignored, examples, hasRelaxedFloat32Model,

                              Executor::ASYNC, MeasureTiming::YES, OutputType::FULLY_SPECIFIED);

        EvaluatePreparedModel(preparedModel, is_ignored, examples, hasRelaxedFloat32Model,

                              Executor::SYNC, MeasureTiming::YES, OutputType::FULLY_SPECIFIED);

        EvaluatePreparedModel(preparedModel, is_ignored, examples, hasRelaxedFloat32Model,

                              Executor::BURST, MeasureTiming::YES, OutputType::FULLY_SPECIFIED);

    }

}



static void getPreparedModel(sp<PreparedModelCallback> callback,

                             sp<V1_0::IPreparedModel>* preparedModel) {

    *preparedModel = callback->getPreparedModel();
@@ -468,12 +514,8 @@ void Execute(const sp<V1_1::IDevice>& device, std::function<V1_1::Model(void)> c 
                          MeasureTiming::NO, OutputType::FULLY_SPECIFIED);

}



// TODO: Reduce code duplication.

void Execute(const sp<V1_2::IDevice>& device, std::function<V1_2::Model(void)> create_model,

             std::function<bool(int)> is_ignored, const std::vector<MixedTypedExample>& examples,

             bool testDynamicOutputShape) {

    V1_2::Model model = create_model();



void PrepareModel(const sp<V1_2::IDevice>& device, const V1_2::Model& model,

                  sp<V1_2::IPreparedModel>* preparedModel) {

    // see if service can handle model

    bool fullySupportsModel = false;

    Return<void> supportedCall = device->getSupportedOperations_1_2(
@@ -496,12 +538,11 @@ void Execute(const sp<V1_2::IDevice>& device, std::function<V1_2::Model(void)> c 
    // retrieve prepared model

    preparedModelCallback->wait();

    ErrorStatus prepareReturnStatus = preparedModelCallback->getStatus();

    sp<V1_2::IPreparedModel> preparedModel;

    getPreparedModel(preparedModelCallback, &preparedModel);

    getPreparedModel(preparedModelCallback, preparedModel);



    // early termination if vendor service cannot fully prepare model

    if (!fullySupportsModel && prepareReturnStatus != ErrorStatus::NONE) {

        ASSERT_EQ(nullptr, preparedModel.get());

        ASSERT_EQ(nullptr, preparedModel->get());

        LOG(INFO) << "NN VTS: Early termination of test because vendor service cannot "

                     "prepare model that it does not support.";

        std::cout << "[          ]   Early termination of test because vendor service cannot "
@@ -510,65 +551,18 @@ void Execute(const sp<V1_2::IDevice>& device, std::function<V1_2::Model(void)> c 
        GTEST_SKIP();

    }

    EXPECT_EQ(ErrorStatus::NONE, prepareReturnStatus);

    ASSERT_NE(nullptr, preparedModel.get());

    ASSERT_NE(nullptr, preparedModel->get());

}



    if (testDynamicOutputShape) {

        EvaluatePreparedModel(preparedModel, is_ignored, examples,

                              model.relaxComputationFloat32toFloat16, Executor::ASYNC,

                              MeasureTiming::NO, OutputType::UNSPECIFIED);

        EvaluatePreparedModel(preparedModel, is_ignored, examples,

                              model.relaxComputationFloat32toFloat16, Executor::SYNC,

                              MeasureTiming::NO, OutputType::UNSPECIFIED);

        EvaluatePreparedModel(preparedModel, is_ignored, examples,

                              model.relaxComputationFloat32toFloat16, Executor::BURST,

                              MeasureTiming::NO, OutputType::UNSPECIFIED);

        EvaluatePreparedModel(preparedModel, is_ignored, examples,

                              model.relaxComputationFloat32toFloat16, Executor::ASYNC,

                              MeasureTiming::YES, OutputType::UNSPECIFIED);

        EvaluatePreparedModel(preparedModel, is_ignored, examples,

                              model.relaxComputationFloat32toFloat16, Executor::SYNC,

                              MeasureTiming::YES, OutputType::UNSPECIFIED);

        EvaluatePreparedModel(preparedModel, is_ignored, examples,

                              model.relaxComputationFloat32toFloat16, Executor::BURST,

                              MeasureTiming::YES, OutputType::UNSPECIFIED);

        EvaluatePreparedModel(preparedModel, is_ignored, examples,

                              model.relaxComputationFloat32toFloat16, Executor::ASYNC,

                              MeasureTiming::NO, OutputType::INSUFFICIENT);

        EvaluatePreparedModel(preparedModel, is_ignored, examples,

                              model.relaxComputationFloat32toFloat16, Executor::SYNC,

                              MeasureTiming::NO, OutputType::INSUFFICIENT);

        EvaluatePreparedModel(preparedModel, is_ignored, examples,

                              model.relaxComputationFloat32toFloat16, Executor::BURST,

                              MeasureTiming::NO, OutputType::INSUFFICIENT);

        EvaluatePreparedModel(preparedModel, is_ignored, examples,

                              model.relaxComputationFloat32toFloat16, Executor::ASYNC,

                              MeasureTiming::YES, OutputType::INSUFFICIENT);

        EvaluatePreparedModel(preparedModel, is_ignored, examples,

                              model.relaxComputationFloat32toFloat16, Executor::SYNC,

                              MeasureTiming::YES, OutputType::INSUFFICIENT);

        EvaluatePreparedModel(preparedModel, is_ignored, examples,

                              model.relaxComputationFloat32toFloat16, Executor::BURST,

                              MeasureTiming::YES, OutputType::INSUFFICIENT);

    } else {

        EvaluatePreparedModel(preparedModel, is_ignored, examples,

                              model.relaxComputationFloat32toFloat16, Executor::ASYNC,

                              MeasureTiming::NO, OutputType::FULLY_SPECIFIED);

        EvaluatePreparedModel(preparedModel, is_ignored, examples,

                              model.relaxComputationFloat32toFloat16, Executor::SYNC,

                              MeasureTiming::NO, OutputType::FULLY_SPECIFIED);

        EvaluatePreparedModel(preparedModel, is_ignored, examples,

                              model.relaxComputationFloat32toFloat16, Executor::BURST,

                              MeasureTiming::NO, OutputType::FULLY_SPECIFIED);

        EvaluatePreparedModel(preparedModel, is_ignored, examples,

                              model.relaxComputationFloat32toFloat16, Executor::ASYNC,

                              MeasureTiming::YES, OutputType::FULLY_SPECIFIED);

        EvaluatePreparedModel(preparedModel, is_ignored, examples,

                              model.relaxComputationFloat32toFloat16, Executor::SYNC,

                              MeasureTiming::YES, OutputType::FULLY_SPECIFIED);

// TODO: Reduce code duplication.

void Execute(const sp<V1_2::IDevice>& device, std::function<V1_2::Model(void)> create_model,

             std::function<bool(int)> is_ignored, const std::vector<MixedTypedExample>& examples,

             bool testDynamicOutputShape) {

    V1_2::Model model = create_model();

    sp<V1_2::IPreparedModel> preparedModel = nullptr;

    PrepareModel(device, model, &preparedModel);

    EvaluatePreparedModel(preparedModel, is_ignored, examples,

                              model.relaxComputationFloat32toFloat16, Executor::BURST,

                              MeasureTiming::YES, OutputType::FULLY_SPECIFIED);

    }

                          model.relaxComputationFloat32toFloat16, testDynamicOutputShape);

}



}  // namespace generated_tests

neuralnetworks/1.0/vts/functional/GeneratedTestHarness.h

0 → 100644
+57 −0
Original line number Diff line number Diff line 
/*

 * Copyright (C) 2019 The Android Open Source Project

 *

 * Licensed under the Apache License, Version 2.0 (the "License");

 * you may not use this file except in compliance with the License.

 * You may obtain a copy of the License at

 *

 *      http://www.apache.org/licenses/LICENSE-2.0

 *

 * Unless required by applicable law or agreed to in writing, software

 * distributed under the License is distributed on an "AS IS" BASIS,

 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

 * See the License for the specific language governing permissions and

 * limitations under the License.

 */



#ifndef VTS_HAL_NEURALNETWORKS_GENERATED_TEST_HARNESS_H

#define VTS_HAL_NEURALNETWORKS_GENERATED_TEST_HARNESS_H



#include "TestHarness.h"



#include <android/hardware/neuralnetworks/1.0/IDevice.h>

#include <android/hardware/neuralnetworks/1.1/IDevice.h>

#include <android/hardware/neuralnetworks/1.2/IDevice.h>



namespace android {

namespace hardware {

namespace neuralnetworks {



namespace generated_tests {

using ::test_helper::MixedTypedExample;



void PrepareModel(const sp<V1_2::IDevice>& device, const V1_2::Model& model,

                  sp<V1_2::IPreparedModel>* preparedModel);



void EvaluatePreparedModel(sp<V1_2::IPreparedModel>& preparedModel,

                           std::function<bool(int)> is_ignored,

                           const std::vector<MixedTypedExample>& examples,

                           bool hasRelaxedFloat32Model, bool testDynamicOutputShape);



void Execute(const sp<V1_0::IDevice>& device, std::function<V1_0::Model(void)> create_model,

             std::function<bool(int)> is_ignored, const std::vector<MixedTypedExample>& examples);



void Execute(const sp<V1_1::IDevice>& device, std::function<V1_1::Model(void)> create_model,

             std::function<bool(int)> is_ignored, const std::vector<MixedTypedExample>& examples);



void Execute(const sp<V1_2::IDevice>& device, std::function<V1_2::Model(void)> create_model,

             std::function<bool(int)> is_ignored, const std::vector<MixedTypedExample>& examples,

             bool testDynamicOutputShape = false);



}  // namespace generated_tests



}  // namespace neuralnetworks

}  // namespace hardware

}  // namespace android



#endif  // VTS_HAL_NEURALNETWORKS_GENERATED_TEST_HARNESS_H

neuralnetworks/1.0/vts/functional/GeneratedTests.cpp

+1 −7
Original line number Diff line number Diff line
@@ -19,6 +19,7 @@ 
#include "VtsHalNeuralnetworks.h"



#include "Callbacks.h"

#include "GeneratedTestHarness.h"

#include "TestHarness.h"

#include "Utils.h"
@@ -29,13 +30,6 @@ 
namespace android {

namespace hardware {

namespace neuralnetworks {



namespace generated_tests {

using ::test_helper::MixedTypedExample;

extern void Execute(const sp<V1_0::IDevice>&, std::function<V1_0::Model(void)>,

                    std::function<bool(int)>, const std::vector<MixedTypedExample>&);

}  // namespace generated_tests



namespace V1_0 {

namespace vts {

namespace functional {

neuralnetworks/1.1/vts/functional/GeneratedTests.cpp

+1 −7
Original line number Diff line number Diff line
@@ -19,6 +19,7 @@ 
#include "VtsHalNeuralnetworks.h"



#include "Callbacks.h"

#include "GeneratedTestHarness.h"

#include "TestHarness.h"

#include "Utils.h"
@@ -29,13 +30,6 @@ 
namespace android {

namespace hardware {

namespace neuralnetworks {



namespace generated_tests {

using ::test_helper::MixedTypedExample;

extern void Execute(const sp<V1_1::IDevice>&, std::function<V1_1::Model(void)>,

                    std::function<bool(int)>, const std::vector<MixedTypedExample>&);

}  // namespace generated_tests



namespace V1_1 {

namespace vts {

namespace functional {

neuralnetworks/1.1/vts/functional/GeneratedTestsV1_0.cpp

+1 −7
Original line number Diff line number Diff line
@@ -19,6 +19,7 @@ 
#include "VtsHalNeuralnetworks.h"



#include "Callbacks.h"

#include "GeneratedTestHarness.h"

#include "TestHarness.h"

#include "Utils.h"
@@ -29,13 +30,6 @@ 
namespace android {

namespace hardware {

namespace neuralnetworks {



namespace generated_tests {

using ::test_helper::MixedTypedExample;

extern void Execute(const sp<V1_1::IDevice>&, std::function<V1_1::Model(void)>,

                    std::function<bool(int)>, const std::vector<MixedTypedExample>&);

}  // namespace generated_tests



namespace V1_1 {

namespace vts {

namespace functional {