Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 7df64ea2 authored by Treehugger Robot's avatar Treehugger Robot Committed by Gerrit Code Review
Browse files

Merge changes from topic "cp_nnapi" 

* changes:
  Relax NeuralNetwork's VTS positive and negative base tests
  Add Capabilities::relaxedFloat32toFloat16Performance for 1.1.
  Minor modification to allow relaxed precision test
  NNAPI narrow evaluation for P -- HAL
  Refactor NN API VTS tests and add v1.1 tests
  Add README to NeuralNetworks HAL
parents 6a83645f 1ae02d63

neuralnetworks/1.0/vts/functional/Android.bp

+31 −4
Original line number Diff line number Diff line
@@ -14,22 +14,49 @@ 
// limitations under the License.

//



cc_test {

    name: "VtsHalNeuralnetworksV1_0TargetTest",

cc_library_static {

    name: "VtsHalNeuralnetworksTest_utils",

    srcs: [

        "Callbacks.cpp",

        "GeneratedTestHarness.cpp",

        "Models.cpp",

        "VtsHalNeuralnetworksV1_0TargetTest.cpp",

        "GeneratedTestHarness.cpp",

    ],

    defaults: ["VtsHalTargetTestDefaults"],

    export_include_dirs: ["."],

    static_libs: [

        "android.hardware.neuralnetworks@1.0",

        "android.hardware.neuralnetworks@1.1",

        "android.hidl.allocator@1.0",

        "android.hidl.memory@1.0",

        "libhidlmemory",

        "libneuralnetworks_utils",

    ],

    header_libs: [

        "libneuralnetworks_headers",

        "libneuralnetworks_generated_test_harness_headers",

        "libneuralnetworks_generated_tests",

    ],

}



cc_test {

    name: "VtsHalNeuralnetworksV1_0TargetTest",

    srcs: [

        "VtsHalNeuralnetworksV1_0.cpp",

        "VtsHalNeuralnetworksV1_0BasicTest.cpp",

        "VtsHalNeuralnetworksV1_0GeneratedTest.cpp",

    ],

    defaults: ["VtsHalTargetTestDefaults"],

    static_libs: [

        "android.hardware.neuralnetworks@1.0",

        "android.hardware.neuralnetworks@1.1",

        "android.hidl.allocator@1.0",

        "android.hidl.memory@1.0",

        "libhidlmemory",

        "libneuralnetworks_utils",

        "VtsHalNeuralnetworksTest_utils",

    ],

    header_libs: [

        "libneuralnetworks_headers",

        "libneuralnetworks_generated_test_harness_headers",

        "libneuralnetworks_generated_tests",

    ],

neuralnetworks/1.0/vts/functional/GeneratedTestHarness.cpp

+104 −58
Original line number Diff line number Diff line
@@ -16,9 +16,15 @@ 


#include "Callbacks.h"

#include "TestHarness.h"

#include "VtsHalNeuralnetworksV1_0TargetTest.h"

#include "Utils.h"



#include <android-base/logging.h>

#include <android/hardware/neuralnetworks/1.0/IDevice.h>

#include <android/hardware/neuralnetworks/1.0/IExecutionCallback.h>

#include <android/hardware/neuralnetworks/1.0/IPreparedModel.h>

#include <android/hardware/neuralnetworks/1.0/IPreparedModelCallback.h>

#include <android/hardware/neuralnetworks/1.0/types.h>

#include <android/hidl/allocator/1.0/IAllocator.h>

#include <android/hidl/memory/1.0/IMemory.h>

#include <hidlmemory/mapping.h>

#include <iostream>
@@ -26,11 +32,6 @@ 
namespace android {

namespace hardware {

namespace neuralnetworks {

namespace V1_0 {

namespace vts {

namespace functional {

// allocator helper

hidl_memory allocateSharedMemory(int64_t size, const std::string& type = "ashmem");



namespace generated_tests {

using ::android::hardware::neuralnetworks::V1_0::implementation::ExecutionCallback;
@@ -64,54 +65,11 @@ void copy_back(MixedTyped* dst, const std::vector<RequestArgument>& ra, char* sr 


// Top level driver for models and examples generated by test_generator.py

// Test driver for those generated from ml/nn/runtime/test/spec

void Execute(const sp<IDevice>& device, std::function<Model(void)> create_model,

             std::function<bool(int)> is_ignored,

             const std::vector<MixedTypedExampleType>& examples) {

void EvaluatePreparedModel(sp<IPreparedModel>& preparedModel, std::function<bool(int)> is_ignored,

                           const std::vector<MixedTypedExampleType>& examples,

                           float fpRange = 1e-5f) {

    const uint32_t INPUT = 0;

    const uint32_t OUTPUT = 1;

    Model model = create_model();



    // see if service can handle model

    ErrorStatus supportedStatus;

    bool fullySupportsModel = false;

    Return<void> supportedCall = device->getSupportedOperations(

        model, [&](ErrorStatus status, const hidl_vec<bool>& supported) {

            supportedStatus = status;

            ASSERT_NE(0ul, supported.size());

            fullySupportsModel =

                std::all_of(supported.begin(), supported.end(), [](bool valid) { return valid; });

        });

    ASSERT_TRUE(supportedCall.isOk());

    ASSERT_EQ(ErrorStatus::NONE, supportedStatus);



    // launch prepare model

    sp<PreparedModelCallback> preparedModelCallback = new PreparedModelCallback();

    ASSERT_NE(nullptr, preparedModelCallback.get());

    Return<ErrorStatus> prepareLaunchStatus = device->prepareModel(model, preparedModelCallback);

    ASSERT_TRUE(prepareLaunchStatus.isOk());



    // retrieve prepared model

    preparedModelCallback->wait();

    ErrorStatus prepareReturnStatus = preparedModelCallback->getStatus();

    sp<IPreparedModel> preparedModel = preparedModelCallback->getPreparedModel();

    if (fullySupportsModel) {

        EXPECT_EQ(ErrorStatus::NONE, prepareReturnStatus);

    } else {

        EXPECT_TRUE(prepareReturnStatus == ErrorStatus::NONE ||

                    prepareReturnStatus == ErrorStatus::GENERAL_FAILURE);

    }



    // early termination if vendor service cannot fully prepare model

    if (!fullySupportsModel && prepareReturnStatus == ErrorStatus::GENERAL_FAILURE) {

        ASSERT_EQ(nullptr, preparedModel.get());

        LOG(INFO) << "NN VTS: Early termination of test because vendor service cannot "

                     "prepare model that it does not support.";

        std::cout << "[          ]   Early termination of test because vendor service cannot "

                     "prepare model that it does not support."

                  << std::endl;

        return;

    }

    ASSERT_NE(nullptr, preparedModel.get());



    int example_no = 1;

    for (auto& example : examples) {
@@ -167,8 +125,8 @@ void Execute(const sp<IDevice>& device, std::function<Model(void)> create_model, 
                offset += i.location.length;

            }

        }

        std::vector<hidl_memory> pools = {allocateSharedMemory(inputSize),

                                          allocateSharedMemory(outputSize)};

        std::vector<hidl_memory> pools = {nn::allocateSharedMemory(inputSize),

                                          nn::allocateSharedMemory(outputSize)};

        ASSERT_NE(0ull, pools[INPUT].size());

        ASSERT_NE(0ull, pools[OUTPUT].size());
@@ -217,15 +175,103 @@ void Execute(const sp<IDevice>& device, std::function<Model(void)> create_model, 
        MixedTyped filtered_test = filter(test, is_ignored);



        // We want "close-enough" results for float

        compare(filtered_golden, filtered_test);

        compare(filtered_golden, filtered_test, fpRange);

    }

}



void Execute(sp<V1_0::IDevice>& device, std::function<V1_0::Model(void)> create_model,

             std::function<bool(int)> is_ignored,

             const std::vector<MixedTypedExampleType>& examples) {

    V1_0::Model model = create_model();



    // see if service can handle model

    bool fullySupportsModel = false;

    Return<void> supportedCall = device->getSupportedOperations(

        model, [&fullySupportsModel](ErrorStatus status, const hidl_vec<bool>& supported) {

            ASSERT_EQ(ErrorStatus::NONE, status);

            ASSERT_NE(0ul, supported.size());

            fullySupportsModel =

                std::all_of(supported.begin(), supported.end(), [](bool valid) { return valid; });

        });

    ASSERT_TRUE(supportedCall.isOk());



    // launch prepare model

    sp<PreparedModelCallback> preparedModelCallback = new PreparedModelCallback();

    ASSERT_NE(nullptr, preparedModelCallback.get());

    Return<ErrorStatus> prepareLaunchStatus = device->prepareModel(model, preparedModelCallback);

    ASSERT_TRUE(prepareLaunchStatus.isOk());

    ASSERT_EQ(ErrorStatus::NONE, static_cast<ErrorStatus>(prepareLaunchStatus));



    // retrieve prepared model

    preparedModelCallback->wait();

    ErrorStatus prepareReturnStatus = preparedModelCallback->getStatus();

    sp<IPreparedModel> preparedModel = preparedModelCallback->getPreparedModel();



    // early termination if vendor service cannot fully prepare model

    if (!fullySupportsModel && prepareReturnStatus != ErrorStatus::NONE) {

        ASSERT_EQ(nullptr, preparedModel.get());

        LOG(INFO) << "NN VTS: Early termination of test because vendor service cannot "

                     "prepare model that it does not support.";

        std::cout << "[          ]   Early termination of test because vendor service cannot "

                     "prepare model that it does not support."

                  << std::endl;

        return;

    }

    EXPECT_EQ(ErrorStatus::NONE, prepareReturnStatus);

    ASSERT_NE(nullptr, preparedModel.get());



    EvaluatePreparedModel(preparedModel, is_ignored, examples);

}



void Execute(sp<V1_1::IDevice>& device, std::function<V1_1::Model(void)> create_model,

             std::function<bool(int)> is_ignored,

             const std::vector<MixedTypedExampleType>& examples) {

    V1_1::Model model = create_model();



    // see if service can handle model

    bool fullySupportsModel = false;

    Return<void> supportedCall = device->getSupportedOperations_1_1(

        model, [&fullySupportsModel](ErrorStatus status, const hidl_vec<bool>& supported) {

            ASSERT_EQ(ErrorStatus::NONE, status);

            ASSERT_NE(0ul, supported.size());

            fullySupportsModel =

                std::all_of(supported.begin(), supported.end(), [](bool valid) { return valid; });

        });

    ASSERT_TRUE(supportedCall.isOk());



    // launch prepare model

    sp<PreparedModelCallback> preparedModelCallback = new PreparedModelCallback();

    ASSERT_NE(nullptr, preparedModelCallback.get());

    Return<ErrorStatus> prepareLaunchStatus =

        device->prepareModel_1_1(model, preparedModelCallback);

    ASSERT_TRUE(prepareLaunchStatus.isOk());

    ASSERT_EQ(ErrorStatus::NONE, static_cast<ErrorStatus>(prepareLaunchStatus));



    // retrieve prepared model

    preparedModelCallback->wait();

    ErrorStatus prepareReturnStatus = preparedModelCallback->getStatus();

    sp<IPreparedModel> preparedModel = preparedModelCallback->getPreparedModel();



    // early termination if vendor service cannot fully prepare model

    if (!fullySupportsModel && prepareReturnStatus != ErrorStatus::NONE) {

        ASSERT_EQ(nullptr, preparedModel.get());

        LOG(INFO) << "NN VTS: Early termination of test because vendor service cannot "

                     "prepare model that it does not support.";

        std::cout << "[          ]   Early termination of test because vendor service cannot "

                     "prepare model that it does not support."

                  << std::endl;

        return;

    }

    EXPECT_EQ(ErrorStatus::NONE, prepareReturnStatus);

    ASSERT_NE(nullptr, preparedModel.get());



    // If in relaxed mode, set the error range to be 5ULP of FP16.

    float fpRange = !model.relaxComputationFloat32toFloat16 ? 1e-5f : 5.0f * 0.0009765625f;

    EvaluatePreparedModel(preparedModel, is_ignored, examples, fpRange);

}



}  // namespace generated_tests



}  // namespace functional

}  // namespace vts

}  // namespace V1_0

}  // namespace neuralnetworks

}  // namespace hardware

}  // namespace android

neuralnetworks/1.0/vts/functional/Models.cpp

+24 −29
Original line number Diff line number Diff line
@@ -17,19 +17,22 @@ 
#define LOG_TAG "neuralnetworks_hidl_hal_test"



#include "Models.h"

#include "Utils.h"



#include <android-base/logging.h>

#include <android/hidl/allocator/1.0/IAllocator.h>

#include <android/hidl/memory/1.0/IMemory.h>

#include <hidlmemory/mapping.h>

#include <vector>



using ::android::sp;



namespace android {

namespace hardware {

namespace neuralnetworks {

namespace V1_0 {

namespace vts {

namespace functional {



// create a valid model

Model createValidTestModel() {

V1_1::Model createValidTestModel_1_1() {

    const std::vector<float> operand2Data = {5.0f, 6.0f, 7.0f, 8.0f};

    const uint32_t size = operand2Data.size() * sizeof(float);
@@ -103,39 +106,34 @@ Model createValidTestModel() { 
}



// create first invalid model

Model createInvalidTestModel1() {

    Model model = createValidTestModel();

V1_1::Model createInvalidTestModel1_1_1() {

    Model model = createValidTestModel_1_1();

    model.operations[0].type = static_cast<OperationType>(0xDEADBEEF); /* INVALID */

    return model;

}



// create second invalid model

Model createInvalidTestModel2() {

    Model model = createValidTestModel();

V1_1::Model createInvalidTestModel2_1_1() {

    Model model = createValidTestModel_1_1();

    const uint32_t operand1 = 0;

    const uint32_t operand5 = 4;  // INVALID OPERAND

    model.inputIndexes = std::vector<uint32_t>({operand1, operand5 /* INVALID OPERAND */});

    return model;

}



// allocator helper

hidl_memory allocateSharedMemory(int64_t size, const std::string& type = "ashmem") {

    hidl_memory memory;



    sp<IAllocator> allocator = IAllocator::getService(type);

    if (!allocator.get()) {

        return {};

V1_0::Model createValidTestModel_1_0() {

    V1_1::Model model = createValidTestModel_1_1();

    return nn::convertToV1_0(model);

}



    Return<void> ret = allocator->allocate(size, [&](bool success, const hidl_memory& mem) {

        ASSERT_TRUE(success);

        memory = mem;

    });

    if (!ret.isOk()) {

        return {};

V1_0::Model createInvalidTestModel1_1_0() {

    V1_1::Model model = createInvalidTestModel1_1_1();

    return nn::convertToV1_0(model);

}



    return memory;

V1_0::Model createInvalidTestModel2_1_0() {

    V1_1::Model model = createInvalidTestModel2_1_1();

    return nn::convertToV1_0(model);

}



// create a valid request
@@ -154,8 +152,8 @@ Request createValidTestRequest() { 
    std::vector<RequestArgument> outputs = {{

        .location = {.poolIndex = OUTPUT, .offset = 0, .length = outputSize}, .dimensions = {},

    }};

    std::vector<hidl_memory> pools = {allocateSharedMemory(inputSize),

                                      allocateSharedMemory(outputSize)};

    std::vector<hidl_memory> pools = {nn::allocateSharedMemory(inputSize),

                                      nn::allocateSharedMemory(outputSize)};

    if (pools[INPUT].size() == 0 || pools[OUTPUT].size() == 0) {

        return {};

    }
@@ -199,9 +197,6 @@ Request createInvalidTestRequest2() { 
    return request;

}



}  // namespace functional

}  // namespace vts

}  // namespace V1_0

}  // namespace neuralnetworks

}  // namespace hardware

}  // namespace android

neuralnetworks/1.0/vts/functional/Models.h

+13 −14
Original line number Diff line number Diff line
@@ -16,28 +16,27 @@ 


#define LOG_TAG "neuralnetworks_hidl_hal_test"



#include "VtsHalNeuralnetworksV1_0TargetTest.h"

#include <android/hardware/neuralnetworks/1.1/types.h>



namespace android {

namespace hardware {

namespace neuralnetworks {

namespace V1_0 {

namespace vts {

namespace functional {



// create the model

Model createValidTestModel();

Model createInvalidTestModel1();

Model createInvalidTestModel2();

// create V1_1 model

V1_1::Model createValidTestModel_1_1();

V1_1::Model createInvalidTestModel1_1_1();

V1_1::Model createInvalidTestModel2_1_1();



// create V1_0 model

V1_0::Model createValidTestModel_1_0();

V1_0::Model createInvalidTestModel1_1_0();

V1_0::Model createInvalidTestModel2_1_0();



// create the request

Request createValidTestRequest();

Request createInvalidTestRequest1();

Request createInvalidTestRequest2();

V1_0::Request createValidTestRequest();

V1_0::Request createInvalidTestRequest1();

V1_0::Request createInvalidTestRequest2();



}  // namespace functional

}  // namespace vts

}  // namespace V1_0

}  // namespace neuralnetworks

}  // namespace hardware

}  // namespace android

neuralnetworks/1.0/vts/functional/VtsHalNeuralnetworksV1_0.cpp

0 → 100644
+73 −0
Original line number Diff line number Diff line 
/*

 * Copyright (C) 2018 The Android Open Source Project

 *

 * Licensed under the Apache License, Version 2.0 (the "License");

 * you may not use this file except in compliance with the License.

 * You may obtain a copy of the License at

 *

 *      http://www.apache.org/licenses/LICENSE-2.0

 *

 * Unless required by applicable law or agreed to in writing, software

 * distributed under the License is distributed on an "AS IS" BASIS,

 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

 * See the License for the specific language governing permissions and

 * limitations under the License.

 */



#define LOG_TAG "neuralnetworks_hidl_hal_test"



#include "VtsHalNeuralnetworksV1_0.h"

#include "Utils.h"



#include <android-base/logging.h>



using ::android::hardware::hidl_memory;

using ::android::hidl::allocator::V1_0::IAllocator;

using ::android::hidl::memory::V1_0::IMemory;

using ::android::sp;



namespace android {

namespace hardware {

namespace neuralnetworks {

namespace V1_0 {

namespace vts {

namespace functional {



// allocator helper

hidl_memory allocateSharedMemory(int64_t size) {

    return nn::allocateSharedMemory(size);

}



// A class for test environment setup

NeuralnetworksHidlEnvironment::NeuralnetworksHidlEnvironment() {}



NeuralnetworksHidlEnvironment::~NeuralnetworksHidlEnvironment() {}



NeuralnetworksHidlEnvironment* NeuralnetworksHidlEnvironment::getInstance() {

    // This has to return a "new" object because it is freed inside

    // ::testing::AddGlobalTestEnvironment when the gtest is being torn down

    static NeuralnetworksHidlEnvironment* instance = new NeuralnetworksHidlEnvironment();

    return instance;

}



void NeuralnetworksHidlEnvironment::registerTestServices() {

    registerTestService<V1_0::IDevice>();

}



// The main test class for NEURALNETWORK HIDL HAL.

NeuralnetworksHidlTest::~NeuralnetworksHidlTest() {}



void NeuralnetworksHidlTest::SetUp() {

    device = ::testing::VtsHalHidlTargetTestBase::getService<V1_0::IDevice>(

        NeuralnetworksHidlEnvironment::getInstance());

    ASSERT_NE(nullptr, device.get());

}



void NeuralnetworksHidlTest::TearDown() {}



}  // namespace functional

}  // namespace vts

}  // namespace V1_0

}  // namespace neuralnetworks

}  // namespace hardware

}  // namespace android