Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 65686222 authored by Xusong Wang's avatar Xusong Wang Committed by Gerrit Code Review
Browse files

Merge changes from topic "nnapi_test_dynamic_reg" 

* changes:
  Statically check test enums against HIDL definitions.
  Update VTS tests with the new test harness.
parents b25a7316 ee80b02a

neuralnetworks/1.0/vts/functional/Android.bp

+3 −2
Original line number Diff line number Diff line
@@ -44,6 +44,7 @@ cc_defaults { 
    name: "VtsHalNeuralNetworksV1_0TargetTestDefaults",

    defaults: ["VtsHalTargetTestDefaults"],

    srcs: [

        "TestAssertions.cpp",

        "ValidateModel.cpp",

        "ValidateRequest.cpp",

        "VtsHalNeuralnetworks.cpp",
@@ -74,7 +75,7 @@ cc_test { 
    defaults: ["VtsHalNeuralNetworksV1_0TargetTestDefaults"],

    srcs: [

        "BasicTests.cpp",

        ":VtsHalNeuralNetworksV1_0_all_generated_V1_0_tests",

        ":VtsHalNeuralNetworksV1_0_all_generated_tests",

    ],

}
@@ -83,7 +84,7 @@ cc_test { 
    defaults: ["VtsHalNeuralNetworksV1_0TargetTestDefaults"],

    srcs: [

        "BasicTests.cpp",

        ":VtsHalNeuralNetworksV1_0_all_generated_V1_0_tests",

        ":VtsHalNeuralNetworksV1_0_all_generated_tests",

    ],

    cflags: [

        "-DPRESUBMIT_NOT_VTS",

neuralnetworks/1.0/vts/functional/GeneratedTestHarness.cpp

+54 −39
Original line number Diff line number Diff line
@@ -20,6 +20,7 @@ 
#include "1.0/Utils.h"

#include "MemoryUtils.h"

#include "TestHarness.h"

#include "VtsHalNeuralnetworks.h"



#include <android-base/logging.h>

#include <android/hardware/neuralnetworks/1.0/IDevice.h>
@@ -36,7 +37,8 @@ namespace android { 
namespace hardware {

namespace neuralnetworks {

namespace V1_0 {

namespace generated_tests {

namespace vts {

namespace functional {



using namespace test_helper;

using ::android::hardware::neuralnetworks::V1_0::ErrorStatus;
@@ -151,7 +153,10 @@ void EvaluatePreparedModel(const sp<IPreparedModel>& preparedModel, const TestMo 
    checkResults(testModel, outputs);

}



void Execute(const sp<IDevice>& device, const TestModel& testModel) {

// Tag for the generated tests

class GeneratedTest : public GeneratedTestBase {

  protected:

    void Execute(const TestModel& testModel) {

        Model model = createModel(testModel);



        // see if service can handle model
@@ -167,7 +172,8 @@ void Execute(const sp<IDevice>& device, const TestModel& testModel) { 


        // launch prepare model

        sp<PreparedModelCallback> preparedModelCallback = new PreparedModelCallback();

    Return<ErrorStatus> prepareLaunchStatus = device->prepareModel(model, preparedModelCallback);

        Return<ErrorStatus> prepareLaunchStatus =

                device->prepareModel(model, preparedModelCallback);

        ASSERT_TRUE(prepareLaunchStatus.isOk());

        ASSERT_EQ(ErrorStatus::NONE, static_cast<ErrorStatus>(prepareLaunchStatus));
@@ -191,8 +197,17 @@ void Execute(const sp<IDevice>& device, const TestModel& testModel) { 


        EvaluatePreparedModel(preparedModel, testModel);

    }

};



}  // namespace generated_tests

TEST_P(GeneratedTest, Test) {

    Execute(*mTestModel);

}



INSTANTIATE_GENERATED_TEST(GeneratedTest,

                           [](const TestModel& testModel) { return !testModel.expectFailure; });



}  // namespace functional

}  // namespace vts

}  // namespace V1_0

}  // namespace neuralnetworks

}  // namespace hardware

neuralnetworks/1.0/vts/functional/GeneratedTestHarness.h

+34 −4
Original line number Diff line number Diff line
@@ -19,18 +19,48 @@ 


#include <android/hardware/neuralnetworks/1.0/IDevice.h>

#include "TestHarness.h"

#include "VtsHalNeuralnetworks.h"



namespace android {

namespace hardware {

namespace neuralnetworks {

namespace V1_0 {

namespace generated_tests {

namespace vts {

namespace functional {



Model createModel(const ::test_helper::TestModel& testModel);

class GeneratedTestBase

    : public NeuralnetworksHidlTest,

      public ::testing::WithParamInterface<test_helper::TestModelManager::TestParam> {

  protected:

    void SetUp() override {

        NeuralnetworksHidlTest::SetUp();

        ASSERT_NE(mTestModel, nullptr);

    }



    const test_helper::TestModel* mTestModel = GetParam().second;

};



#define INSTANTIATE_GENERATED_TEST(TestSuite, filter)                                          \

    INSTANTIATE_TEST_SUITE_P(                                                                  \

            TestGenerated, TestSuite,                                                          \

            ::testing::ValuesIn(::test_helper::TestModelManager::get().getTestModels(filter)), \

            [](const auto& info) { return info.param.first; })



void Execute(const sp<V1_0::IDevice>& device, const ::test_helper::TestModel& testModel);

// Tag for the validation tests, instantiated in VtsHalNeuralnetworks.cpp.

// TODO: Clean up the hierarchy for ValidationTest.

class ValidationTest : public GeneratedTestBase {

  protected:

    void validateEverything(const Model& model, const Request& request);



  private:

    void validateModel(const Model& model);

    void validateRequest(const sp<IPreparedModel>& preparedModel, const Request& request);

};



Model createModel(const ::test_helper::TestModel& testModel);



}  // namespace generated_tests

}  // namespace functional

}  // namespace vts

}  // namespace V1_0

}  // namespace neuralnetworks

}  // namespace hardware

neuralnetworks/1.0/vts/functional/GeneratedTests.h

deleted100644 → 0
+0 −26
Original line number Diff line number Diff line 
/*

 * Copyright (C) 2019 The Android Open Source Project

 *

 * Licensed under the Apache License, Version 2.0 (the "License");

 * you may not use this file except in compliance with the License.

 * You may obtain a copy of the License at

 *

 *      http://www.apache.org/licenses/LICENSE-2.0

 *

 * Unless required by applicable law or agreed to in writing, software

 * distributed under the License is distributed on an "AS IS" BASIS,

 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

 * See the License for the specific language governing permissions and

 * limitations under the License.

 */



#include "1.0/Utils.h"

#include "GeneratedTestHarness.h"

#include "TestHarness.h"

#include "VtsHalNeuralnetworks.h"



namespace android::hardware::neuralnetworks::V1_0::generated_tests {



using namespace android::hardware::neuralnetworks::V1_0::vts::functional;



}  // namespace android::hardware::neuralnetworks::V1_0::generated_tests

neuralnetworks/1.0/vts/functional/TestAssertions.cpp

0 → 100644
+74 −0
Original line number Diff line number Diff line 
/*

 * Copyright (C) 2019 The Android Open Source Project

 *

 * Licensed under the Apache License, Version 2.0 (the "License");

 * you may not use this file except in compliance with the License.

 * You may obtain a copy of the License at

 *

 *      http://www.apache.org/licenses/LICENSE-2.0

 *

 * Unless required by applicable law or agreed to in writing, software

 * distributed under the License is distributed on an "AS IS" BASIS,

 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

 * See the License for the specific language governing permissions and

 * limitations under the License.

 */



#include <android/hardware/neuralnetworks/1.0/types.h>

#include "TestHarness.h"



namespace android::hardware::neuralnetworks::V1_0 {



// Make sure that the HIDL enums are compatible with the values defined in

// frameworks/ml/nn/tools/test_generator/test_harness/include/TestHarness.h.

using namespace test_helper;

#define CHECK_TEST_ENUM(EnumType, enumValue) \

    static_assert(static_cast<EnumType>(Test##EnumType::enumValue) == EnumType::enumValue)



CHECK_TEST_ENUM(OperandType, FLOAT32);

CHECK_TEST_ENUM(OperandType, INT32);

CHECK_TEST_ENUM(OperandType, UINT32);

CHECK_TEST_ENUM(OperandType, TENSOR_FLOAT32);

CHECK_TEST_ENUM(OperandType, TENSOR_INT32);

CHECK_TEST_ENUM(OperandType, TENSOR_QUANT8_ASYMM);



CHECK_TEST_ENUM(OperandLifeTime, TEMPORARY_VARIABLE);

CHECK_TEST_ENUM(OperandLifeTime, MODEL_INPUT);

CHECK_TEST_ENUM(OperandLifeTime, MODEL_OUTPUT);

CHECK_TEST_ENUM(OperandLifeTime, CONSTANT_COPY);

CHECK_TEST_ENUM(OperandLifeTime, CONSTANT_REFERENCE);

CHECK_TEST_ENUM(OperandLifeTime, NO_VALUE);



CHECK_TEST_ENUM(OperationType, ADD);

CHECK_TEST_ENUM(OperationType, AVERAGE_POOL_2D);

CHECK_TEST_ENUM(OperationType, CONCATENATION);

CHECK_TEST_ENUM(OperationType, CONV_2D);

CHECK_TEST_ENUM(OperationType, DEPTHWISE_CONV_2D);

CHECK_TEST_ENUM(OperationType, DEPTH_TO_SPACE);

CHECK_TEST_ENUM(OperationType, DEQUANTIZE);

CHECK_TEST_ENUM(OperationType, EMBEDDING_LOOKUP);

CHECK_TEST_ENUM(OperationType, FLOOR);

CHECK_TEST_ENUM(OperationType, FULLY_CONNECTED);

CHECK_TEST_ENUM(OperationType, HASHTABLE_LOOKUP);

CHECK_TEST_ENUM(OperationType, L2_NORMALIZATION);

CHECK_TEST_ENUM(OperationType, L2_POOL_2D);

CHECK_TEST_ENUM(OperationType, LOCAL_RESPONSE_NORMALIZATION);

CHECK_TEST_ENUM(OperationType, LOGISTIC);

CHECK_TEST_ENUM(OperationType, LSH_PROJECTION);

CHECK_TEST_ENUM(OperationType, LSTM);

CHECK_TEST_ENUM(OperationType, MAX_POOL_2D);

CHECK_TEST_ENUM(OperationType, MUL);

CHECK_TEST_ENUM(OperationType, RELU);

CHECK_TEST_ENUM(OperationType, RELU1);

CHECK_TEST_ENUM(OperationType, RELU6);

CHECK_TEST_ENUM(OperationType, RESHAPE);

CHECK_TEST_ENUM(OperationType, RESIZE_BILINEAR);

CHECK_TEST_ENUM(OperationType, RNN);

CHECK_TEST_ENUM(OperationType, SOFTMAX);

CHECK_TEST_ENUM(OperationType, SPACE_TO_DEPTH);

CHECK_TEST_ENUM(OperationType, SVDF);

CHECK_TEST_ENUM(OperationType, TANH);



#undef CHECK_TEST_ENUM



}  // namespace android::hardware::neuralnetworks::V1_0