Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 7ed61354 authored by Michael Butler's avatar Michael Butler
Browse files

NN validation tests 

This CL adds validation tests for all of the existing generated models.
The strategy of this CL is this: given a valid model or request, make a
single change to invalidate the model or request, then verify that the
vendor service driver catches the inconsistency and returns
INVALID_ARGUMENT.

Bug: 67828197
Test: mma
Test: VtsHalNeuralnetworksV1_0TargetTest
Test: VtsHalNeuralnetworksV1_1TargetTest
Merged-In: I8efcdbdccc77aaf78992e52c1eac5c940fc81a03
Change-Id: I8efcdbdccc77aaf78992e52c1eac5c940fc81a03
(cherry picked from commit f76acd03)
parent fa89f009

neuralnetworks/1.0/vts/functional/Android.bp

+7 −5
Original line number Diff line number Diff line
@@ -18,7 +18,6 @@ cc_library_static { 
    name: "VtsHalNeuralnetworksTest_utils",

    srcs: [

        "Callbacks.cpp",

        "Models.cpp",

        "GeneratedTestHarness.cpp",

    ],

    defaults: ["VtsHalTargetTestDefaults"],
@@ -41,14 +40,17 @@ cc_library_static { 
cc_test {

    name: "VtsHalNeuralnetworksV1_0TargetTest",

    srcs: [

        "VtsHalNeuralnetworksV1_0.cpp",

        "VtsHalNeuralnetworksV1_0BasicTest.cpp",

        "VtsHalNeuralnetworksV1_0GeneratedTest.cpp",

        "BasicTests.cpp",

        "GeneratedTests.cpp",

        "ValidateModel.cpp",

        "ValidateRequest.cpp",

        "ValidationTests.cpp",

        "VtsHalNeuralnetworks.cpp",

    ],

    defaults: ["VtsHalTargetTestDefaults"],

    static_libs: [

        "android.hardware.neuralnetworks@1.0",

        "android.hardware.neuralnetworks@1.1",

        "android.hardware.neuralnetworks@1.0",

        "android.hidl.allocator@1.0",

        "android.hidl.memory@1.0",

        "libhidlmemory",

neuralnetworks/1.0/vts/functional/BasicTests.cpp

0 → 100644
+56 −0
Original line number Diff line number Diff line 
/*

 * Copyright (C) 2018 The Android Open Source Project

 *

 * Licensed under the Apache License, Version 2.0 (the "License");

 * you may not use this file except in compliance with the License.

 * You may obtain a copy of the License at

 *

 *      http://www.apache.org/licenses/LICENSE-2.0

 *

 * Unless required by applicable law or agreed to in writing, software

 * distributed under the License is distributed on an "AS IS" BASIS,

 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

 * See the License for the specific language governing permissions and

 * limitations under the License.

 */



#define LOG_TAG "neuralnetworks_hidl_hal_test"



#include "VtsHalNeuralnetworks.h"



namespace android {

namespace hardware {

namespace neuralnetworks {

namespace V1_0 {

namespace vts {

namespace functional {



// create device test

TEST_F(NeuralnetworksHidlTest, CreateDevice) {}



// status test

TEST_F(NeuralnetworksHidlTest, StatusTest) {

    Return<DeviceStatus> status = device->getStatus();

    ASSERT_TRUE(status.isOk());

    EXPECT_EQ(DeviceStatus::AVAILABLE, static_cast<DeviceStatus>(status));

}



// initialization

TEST_F(NeuralnetworksHidlTest, GetCapabilitiesTest) {

    Return<void> ret =

        device->getCapabilities([](ErrorStatus status, const Capabilities& capabilities) {

            EXPECT_EQ(ErrorStatus::NONE, status);

            EXPECT_LT(0.0f, capabilities.float32Performance.execTime);

            EXPECT_LT(0.0f, capabilities.float32Performance.powerUsage);

            EXPECT_LT(0.0f, capabilities.quantized8Performance.execTime);

            EXPECT_LT(0.0f, capabilities.quantized8Performance.powerUsage);

        });

    EXPECT_TRUE(ret.isOk());

}



}  // namespace functional

}  // namespace vts

}  // namespace V1_0

}  // namespace neuralnetworks

}  // namespace hardware

}  // namespace android

neuralnetworks/1.0/vts/functional/Callbacks.h

+0 −8
Original line number Diff line number Diff line
@@ -17,14 +17,6 @@ namespace neuralnetworks { 
namespace V1_0 {

namespace implementation {



using ::android::hardware::hidl_array;

using ::android::hardware::hidl_memory;

using ::android::hardware::hidl_string;

using ::android::hardware::hidl_vec;

using ::android::hardware::Return;

using ::android::hardware::Void;

using ::android::sp;



/**

 * The CallbackBase class is used internally by the NeuralNetworks runtime to

 * synchronize between different threads. An asynchronous task is launched

neuralnetworks/1.0/vts/functional/GeneratedTestHarness.cpp

+2 −2
Original line number Diff line number Diff line
@@ -179,7 +179,7 @@ void EvaluatePreparedModel(sp<IPreparedModel>& preparedModel, std::function<bool 
    }

}



void Execute(sp<V1_0::IDevice>& device, std::function<V1_0::Model(void)> create_model,

void Execute(const sp<V1_0::IDevice>& device, std::function<V1_0::Model(void)> create_model,

             std::function<bool(int)> is_ignored,

             const std::vector<MixedTypedExampleType>& examples) {

    V1_0::Model model = create_model();
@@ -223,7 +223,7 @@ void Execute(sp<V1_0::IDevice>& device, std::function<V1_0::Model(void)> create_ 
    EvaluatePreparedModel(preparedModel, is_ignored, examples);

}



void Execute(sp<V1_1::IDevice>& device, std::function<V1_1::Model(void)> create_model,

void Execute(const sp<V1_1::IDevice>& device, std::function<V1_1::Model(void)> create_model,

             std::function<bool(int)> is_ignored,

             const std::vector<MixedTypedExampleType>& examples) {

    V1_1::Model model = create_model();

neuralnetworks/1.0/vts/functional/VtsHalNeuralnetworksV1_0GeneratedTest.cpp→neuralnetworks/1.0/vts/functional/GeneratedTests.cpp

+6 −20
Original line number Diff line number Diff line
@@ -16,47 +16,33 @@ 


#define LOG_TAG "neuralnetworks_hidl_hal_test"



#include "VtsHalNeuralnetworksV1_0.h"

#include "VtsHalNeuralnetworks.h"



#include "Callbacks.h"

#include "TestHarness.h"

#include "Utils.h"



#include <android-base/logging.h>

#include <android/hidl/memory/1.0/IMemory.h>

#include <hidlmemory/mapping.h>



using ::android::hardware::neuralnetworks::V1_0::IDevice;

using ::android::hardware::neuralnetworks::V1_0::IPreparedModel;

using ::android::hardware::neuralnetworks::V1_0::Capabilities;

using ::android::hardware::neuralnetworks::V1_0::DeviceStatus;

using ::android::hardware::neuralnetworks::V1_0::FusedActivationFunc;

using ::android::hardware::neuralnetworks::V1_0::Model;

using ::android::hardware::neuralnetworks::V1_0::OperationType;

using ::android::hardware::neuralnetworks::V1_0::PerformanceInfo;

using ::android::hardware::Return;

using ::android::hardware::Void;

using ::android::hardware::hidl_memory;

using ::android::hardware::hidl_string;

using ::android::hardware::hidl_vec;

using ::android::hidl::allocator::V1_0::IAllocator;

using ::android::hidl::memory::V1_0::IMemory;

using ::android::sp;



namespace android {

namespace hardware {

namespace neuralnetworks {



namespace generated_tests {

using ::generated_tests::MixedTypedExampleType;

extern void Execute(sp<IDevice>&, std::function<Model(void)>, std::function<bool(int)>,

                    const std::vector<MixedTypedExampleType>&);

extern void Execute(const sp<V1_0::IDevice>&, std::function<V1_0::Model(void)>,

                    std::function<bool(int)>, const std::vector<MixedTypedExampleType>&);

}  // namespace generated_tests



namespace V1_0 {

namespace vts {

namespace functional {



using ::android::hardware::neuralnetworks::V1_0::implementation::ExecutionCallback;

using ::android::hardware::neuralnetworks::V1_0::implementation::PreparedModelCallback;

using ::android::nn::allocateSharedMemory;



// Mixed-typed examples

typedef generated_tests::MixedTypedExampleType MixedTypedExample;