Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 941d61a6 authored by Michael K. Sanders's avatar Michael K. Sanders
Browse files

Implements RANDOM_MULTINOMIAL CTS/VTS tests. 

Bug: 113562775
Test: VtsHalNeuralnetworksV1_2TargetTest --hal_service_instance=android.hardware.neuralnetworks@1.2::IDevice/sample-all
Change-Id: I3d39d056bf5034e01ce9c5c5e3f416a63d7ec8eb
parent 6e457995

neuralnetworks/1.0/vts/functional/Android.bp

+2 −0
Original line number Diff line number Diff line
@@ -28,6 +28,7 @@ cc_library_static { 
        "android.hardware.neuralnetworks@1.2",

        "android.hidl.allocator@1.0",

        "android.hidl.memory@1.0",

        "libgmock",

        "libhidlmemory",

        "libneuralnetworks_utils",

    ],
@@ -55,6 +56,7 @@ cc_test { 
        "android.hardware.neuralnetworks@1.2",

        "android.hidl.allocator@1.0",

        "android.hidl.memory@1.0",

        "libgmock",

        "libhidlmemory",

        "libneuralnetworks_utils",

        "VtsHalNeuralnetworksTest_utils",

neuralnetworks/1.0/vts/functional/GeneratedTestHarness.cpp

+16 −14
Original line number Diff line number Diff line
@@ -36,16 +36,17 @@ namespace neuralnetworks { 
namespace generated_tests {

using ::android::hardware::neuralnetworks::V1_0::implementation::ExecutionCallback;

using ::android::hardware::neuralnetworks::V1_0::implementation::PreparedModelCallback;

using ::test_helper::compare;

using ::test_helper::expectMultinomialDistributionWithinTolerance;

using ::test_helper::filter;

using ::test_helper::Float32Operands;

using ::test_helper::for_all;

using ::test_helper::for_each;

using ::test_helper::resize_accordingly;

using ::test_helper::MixedTyped;

using ::test_helper::MixedTypedExampleType;

using ::test_helper::Float32Operands;

using ::test_helper::Int32Operands;

using ::test_helper::MixedTyped;

using ::test_helper::MixedTypedExample;

using ::test_helper::Quant8Operands;

using ::test_helper::compare;

using ::test_helper::resize_accordingly;



template <typename T>

void copy_back_(MixedTyped* dst, const std::vector<RequestArgument>& ra, char* src) {
@@ -66,7 +67,7 @@ void copy_back(MixedTyped* dst, const std::vector<RequestArgument>& ra, char* sr 
// Top level driver for models and examples generated by test_generator.py

// Test driver for those generated from ml/nn/runtime/test/spec

void EvaluatePreparedModel(sp<IPreparedModel>& preparedModel, std::function<bool(int)> is_ignored,

                           const std::vector<MixedTypedExampleType>& examples, float fpAtol = 1e-5f,

                           const std::vector<MixedTypedExample>& examples, float fpAtol = 1e-5f,

                           float fpRtol = 1e-5f) {

    const uint32_t INPUT = 0;

    const uint32_t OUTPUT = 1;
@@ -75,8 +76,8 @@ void EvaluatePreparedModel(sp<IPreparedModel>& preparedModel, std::function<bool 
    for (auto& example : examples) {

        SCOPED_TRACE(example_no++);



        const MixedTyped& inputs = example.first;

        const MixedTyped& golden = example.second;

        const MixedTyped& inputs = example.operands.first;

        const MixedTyped& golden = example.operands.second;



        std::vector<RequestArgument> inputs_info, outputs_info;

        uint32_t inputSize = 0, outputSize = 0;
@@ -176,12 +177,15 @@ void EvaluatePreparedModel(sp<IPreparedModel>& preparedModel, std::function<bool 


        // We want "close-enough" results for float

        compare(filtered_golden, filtered_test, fpAtol, fpRtol);



        if (example.expectedMultinomialDistributionTolerance > 0) {

            expectMultinomialDistributionWithinTolerance(test, example);

        }

    }

}



void Execute(const sp<V1_0::IDevice>& device, std::function<V1_0::Model(void)> create_model,

             std::function<bool(int)> is_ignored,

             const std::vector<MixedTypedExampleType>& examples) {

             std::function<bool(int)> is_ignored, const std::vector<MixedTypedExample>& examples) {

    V1_0::Model model = create_model();



    // see if service can handle model
@@ -225,8 +229,7 @@ void Execute(const sp<V1_0::IDevice>& device, std::function<V1_0::Model(void)> c 
}



void Execute(const sp<V1_1::IDevice>& device, std::function<V1_1::Model(void)> create_model,

             std::function<bool(int)> is_ignored,

             const std::vector<MixedTypedExampleType>& examples) {

             std::function<bool(int)> is_ignored, const std::vector<MixedTypedExample>& examples) {

    V1_1::Model model = create_model();



    // see if service can handle model
@@ -277,8 +280,7 @@ void Execute(const sp<V1_1::IDevice>& device, std::function<V1_1::Model(void)> c 


// TODO: Reduce code duplication.

void Execute(const sp<V1_2::IDevice>& device, std::function<V1_2::Model(void)> create_model,

             std::function<bool(int)> is_ignored,

             const std::vector<MixedTypedExampleType>& examples) {

             std::function<bool(int)> is_ignored, const std::vector<MixedTypedExample>& examples) {

    V1_2::Model model = create_model();



    // see if service can handle model

neuralnetworks/1.0/vts/functional/GeneratedTests.cpp

+3 −5
Original line number Diff line number Diff line
@@ -31,9 +31,9 @@ namespace hardware { 
namespace neuralnetworks {



namespace generated_tests {

using ::test_helper::MixedTypedExampleType;

using ::test_helper::MixedTypedExample;

extern void Execute(const sp<V1_0::IDevice>&, std::function<V1_0::Model(void)>,

                    std::function<bool(int)>, const std::vector<MixedTypedExampleType>&);

                    std::function<bool(int)>, const std::vector<MixedTypedExample>&);

}  // namespace generated_tests



namespace V1_0 {
@@ -43,9 +43,7 @@ namespace functional { 
using ::android::hardware::neuralnetworks::V1_0::implementation::ExecutionCallback;

using ::android::hardware::neuralnetworks::V1_0::implementation::PreparedModelCallback;

using ::android::nn::allocateSharedMemory;



// Mixed-typed examples

typedef test_helper::MixedTypedExampleType MixedTypedExample;

using ::test_helper::MixedTypedExample;



// in frameworks/ml/nn/runtime/tests/generated/

#include "all_generated_V1_0_vts_tests.cpp"

neuralnetworks/1.0/vts/functional/Models.h

+1 −1
Original line number Diff line number Diff line
@@ -30,7 +30,7 @@ namespace V1_0 { 
namespace vts {

namespace functional {



using MixedTypedExample = test_helper::MixedTypedExampleType;

using MixedTypedExample = test_helper::MixedTypedExample;



#define FOR_EACH_TEST_MODEL(FN)                          \

    FN(add_broadcast_quant8)                             \

neuralnetworks/1.0/vts/functional/ValidateRequest.cpp

+6 −6
Original line number Diff line number Diff line
@@ -36,9 +36,9 @@ namespace functional { 
using ::android::hardware::neuralnetworks::V1_0::implementation::ExecutionCallback;

using ::android::hardware::neuralnetworks::V1_0::implementation::PreparedModelCallback;

using ::android::hidl::memory::V1_0::IMemory;

using test_helper::MixedTyped;

using test_helper::MixedTypedExampleType;

using test_helper::for_all;

using test_helper::MixedTyped;

using test_helper::MixedTypedExample;



///////////////////////// UTILITY FUNCTIONS /////////////////////////
@@ -151,15 +151,15 @@ static void removeOutputTest(const sp<IPreparedModel>& preparedModel, const Requ 


///////////////////////////// ENTRY POINT //////////////////////////////////



std::vector<Request> createRequests(const std::vector<MixedTypedExampleType>& examples) {

std::vector<Request> createRequests(const std::vector<MixedTypedExample>& examples) {

    const uint32_t INPUT = 0;

    const uint32_t OUTPUT = 1;



    std::vector<Request> requests;



    for (auto& example : examples) {

        const MixedTyped& inputs = example.first;

        const MixedTyped& outputs = example.second;

    for (const MixedTypedExample& example : examples) {

        const MixedTyped& inputs = example.operands.first;

        const MixedTyped& outputs = example.operands.second;



        std::vector<RequestArgument> inputs_info, outputs_info;

        uint32_t inputSize = 0, outputSize = 0;