Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 0c03b850 authored by Xusong Wang's avatar Xusong Wang
Browse files

Modify 1.1 VTS tests to consume test struct directly. 

This CL is very similar to the 1.0 VTS CL. The only difference is that
1.1 Models have an additional field for relaxed model computation.

Bug: 123092187
Bug: 138718240
Test: All VTS
Change-Id: I9264e5b2468b9c6db47d86683d24f8c2c5ec46aa
Merged-In: I9264e5b2468b9c6db47d86683d24f8c2c5ec46aa
(cherry picked from commit 6aad0402)
parent 8487c5e2

neuralnetworks/1.1/vts/functional/Android.bp

+1 −2
Original line number Diff line number Diff line
@@ -34,13 +34,12 @@ cc_defaults { 
        "android.hidl.memory@1.0",

        "libgmock",

        "libhidlmemory",

        "libneuralnetworks_generated_test_harness",

        "libneuralnetworks_utils",

        "VtsHalNeuralNetworksV1_0_utils",

    ],

    header_libs: [

        "libneuralnetworks_headers",

        "libneuralnetworks_generated_test_harness_headers",

        "libneuralnetworks_generated_tests",

    ],

    test_suites: ["general-tests"],

}

neuralnetworks/1.1/vts/functional/GeneratedTestHarness.cpp

+103 −131
Original line number Diff line number Diff line
@@ -24,6 +24,7 @@ 
#include <android/hidl/memory/1.0/IMemory.h>

#include <hidlmemory/mapping.h>



#include <gtest/gtest.h>

#include <iostream>



#include "1.0/Callbacks.h"
@@ -37,8 +38,13 @@ namespace neuralnetworks { 
namespace V1_1 {

namespace generated_tests {



using namespace test_helper;

using ::android::hardware::neuralnetworks::V1_0::DataLocation;

using ::android::hardware::neuralnetworks::V1_0::ErrorStatus;

using ::android::hardware::neuralnetworks::V1_0::IPreparedModel;

using ::android::hardware::neuralnetworks::V1_0::Operand;

using ::android::hardware::neuralnetworks::V1_0::OperandLifeTime;

using ::android::hardware::neuralnetworks::V1_0::OperandType;

using ::android::hardware::neuralnetworks::V1_0::Request;

using ::android::hardware::neuralnetworks::V1_0::RequestArgument;

using ::android::hardware::neuralnetworks::V1_0::implementation::ExecutionCallback;
@@ -47,144 +53,112 @@ using ::android::hardware::neuralnetworks::V1_1::ExecutionPreference; 
using ::android::hardware::neuralnetworks::V1_1::IDevice;

using ::android::hardware::neuralnetworks::V1_1::Model;

using ::android::hidl::memory::V1_0::IMemory;

using ::test_helper::compare;

using ::test_helper::filter;

using ::test_helper::for_all;

using ::test_helper::MixedTyped;

using ::test_helper::MixedTypedExample;

using ::test_helper::resize_accordingly;



// Top level driver for models and examples generated by test_generator.py

// Test driver for those generated from ml/nn/runtime/test/spec

void EvaluatePreparedModel(sp<IPreparedModel>& preparedModel, std::function<bool(int)> is_ignored,

                           const std::vector<MixedTypedExample>& examples,

                           bool hasRelaxedFloat32Model, float fpAtol, float fpRtol) {

    const uint32_t INPUT = 0;

    const uint32_t OUTPUT = 1;



    int example_no = 1;

    for (auto& example : examples) {

        SCOPED_TRACE(example_no++);

        const MixedTyped& inputs = example.operands.first;

        const MixedTyped& golden = example.operands.second;



        const bool hasFloat16Inputs = !inputs.float16Operands.empty();

        if (hasRelaxedFloat32Model || hasFloat16Inputs) {

            // TODO: Adjust the error limit based on testing.

            // If in relaxed mode, set the absolute tolerance to be 5ULP of FP16.

            fpAtol = 5.0f * 0.0009765625f;

            // Set the relative tolerance to be 5ULP of the corresponding FP precision.

            fpRtol = 5.0f * 0.0009765625f;

Model createModel(const TestModel& testModel) {

    // Model operands.

    hidl_vec<Operand> operands(testModel.operands.size());

    size_t constCopySize = 0, constRefSize = 0;

    for (uint32_t i = 0; i < testModel.operands.size(); i++) {

        const auto& op = testModel.operands[i];



        DataLocation loc = {};

        if (op.lifetime == TestOperandLifeTime::CONSTANT_COPY) {

            loc = {.poolIndex = 0,

                   .offset = static_cast<uint32_t>(constCopySize),

                   .length = static_cast<uint32_t>(op.data.size())};

            constCopySize += op.data.alignedSize();

        } else if (op.lifetime == TestOperandLifeTime::CONSTANT_REFERENCE) {

            loc = {.poolIndex = 0,

                   .offset = static_cast<uint32_t>(constRefSize),

                   .length = static_cast<uint32_t>(op.data.size())};

            constRefSize += op.data.alignedSize();

        }



        std::vector<RequestArgument> inputs_info, outputs_info;

        uint32_t inputSize = 0, outputSize = 0;

        // This function only partially specifies the metadata (vector of RequestArguments).

        // The contents are copied over below.

        for_all(inputs, [&inputs_info, &inputSize](int index, auto, auto s) {

            if (inputs_info.size() <= static_cast<size_t>(index)) inputs_info.resize(index + 1);

            RequestArgument arg = {

                    .location = {.poolIndex = INPUT,

                                 .offset = 0,

                                 .length = static_cast<uint32_t>(s)},

                    .dimensions = {},

            };

            RequestArgument arg_empty = {

                    .hasNoValue = true,

            };

            inputs_info[index] = s ? arg : arg_empty;

            inputSize += s;

        });

        // Compute offset for inputs 1 and so on

        {

            size_t offset = 0;

            for (auto& i : inputs_info) {

                if (!i.hasNoValue) i.location.offset = offset;

                offset += i.location.length;

            }

        operands[i] = {.type = static_cast<OperandType>(op.type),

                       .dimensions = op.dimensions,

                       .numberOfConsumers = op.numberOfConsumers,

                       .scale = op.scale,

                       .zeroPoint = op.zeroPoint,

                       .lifetime = static_cast<OperandLifeTime>(op.lifetime),

                       .location = loc};

    }



        MixedTyped test;  // holding test results



        // Go through all outputs, initialize RequestArgument descriptors

        resize_accordingly(golden, test);

        for_all(golden, [&outputs_info, &outputSize](int index, auto, auto s) {

            if (outputs_info.size() <= static_cast<size_t>(index)) outputs_info.resize(index + 1);

            RequestArgument arg = {

                    .location = {.poolIndex = OUTPUT,

                                 .offset = 0,

                                 .length = static_cast<uint32_t>(s)},

                    .dimensions = {},

            };

            outputs_info[index] = arg;

            outputSize += s;

    // Model operations.

    hidl_vec<Operation> operations(testModel.operations.size());

    std::transform(testModel.operations.begin(), testModel.operations.end(), operations.begin(),

                   [](const TestOperation& op) -> Operation {

                       return {.type = static_cast<OperationType>(op.type),

                               .inputs = op.inputs,

                               .outputs = op.outputs};

                   });

        // Compute offset for outputs 1 and so on

        {

            size_t offset = 0;

            for (auto& i : outputs_info) {

                i.location.offset = offset;

                offset += i.location.length;



    // Constant copies.

    hidl_vec<uint8_t> operandValues(constCopySize);

    for (uint32_t i = 0; i < testModel.operands.size(); i++) {

        const auto& op = testModel.operands[i];

        if (op.lifetime == TestOperandLifeTime::CONSTANT_COPY) {

            const uint8_t* begin = op.data.get<uint8_t>();

            const uint8_t* end = begin + op.data.size();

            std::copy(begin, end, operandValues.data() + operands[i].location.offset);

        }

    }

        std::vector<hidl_memory> pools = {nn::allocateSharedMemory(inputSize),

                                          nn::allocateSharedMemory(outputSize)};

        ASSERT_NE(0ull, pools[INPUT].size());

        ASSERT_NE(0ull, pools[OUTPUT].size());



    // Shared memory.

    hidl_vec<hidl_memory> pools;

    if (constRefSize > 0) {

        hidl_vec_push_back(&pools, nn::allocateSharedMemory(constRefSize));

        CHECK_NE(pools[0].size(), 0u);



        // load data

        sp<IMemory> inputMemory = mapMemory(pools[INPUT]);

        sp<IMemory> outputMemory = mapMemory(pools[OUTPUT]);

        ASSERT_NE(nullptr, inputMemory.get());

        ASSERT_NE(nullptr, outputMemory.get());

        char* inputPtr = reinterpret_cast<char*>(static_cast<void*>(inputMemory->getPointer()));

        char* outputPtr = reinterpret_cast<char*>(static_cast<void*>(outputMemory->getPointer()));

        ASSERT_NE(nullptr, inputPtr);

        ASSERT_NE(nullptr, outputPtr);

        inputMemory->update();

        outputMemory->update();



        // Go through all inputs, copy the values

        for_all(inputs, [&inputs_info, inputPtr](int index, auto p, auto s) {

            char* begin = (char*)p;

            char* end = begin + s;

            // TODO: handle more than one input

            std::copy(begin, end, inputPtr + inputs_info[index].location.offset);

        });

        sp<IMemory> mappedMemory = mapMemory(pools[0]);

        CHECK(mappedMemory.get() != nullptr);

        uint8_t* mappedPtr =

                reinterpret_cast<uint8_t*>(static_cast<void*>(mappedMemory->getPointer()));

        CHECK(mappedPtr != nullptr);



        for (uint32_t i = 0; i < testModel.operands.size(); i++) {

            const auto& op = testModel.operands[i];

            if (op.lifetime == TestOperandLifeTime::CONSTANT_REFERENCE) {

                const uint8_t* begin = op.data.get<uint8_t>();

                const uint8_t* end = begin + op.data.size();

                std::copy(begin, end, mappedPtr + operands[i].location.offset);

            }

        }

    }



        inputMemory->commit();

        outputMemory->commit();

    return {.operands = std::move(operands),

            .operations = std::move(operations),

            .inputIndexes = testModel.inputIndexes,

            .outputIndexes = testModel.outputIndexes,

            .operandValues = std::move(operandValues),

            .pools = std::move(pools),

            .relaxComputationFloat32toFloat16 = testModel.isRelaxed};

}



        const Request request = {.inputs = inputs_info, .outputs = outputs_info, .pools = pools};

// Top level driver for models and examples generated by test_generator.py

// Test driver for those generated from ml/nn/runtime/test/spec

void EvaluatePreparedModel(const sp<IPreparedModel>& preparedModel, const TestModel& testModel) {

    const Request request = createRequest(testModel);



        // launch execution

    // Launch execution.

    sp<ExecutionCallback> executionCallback = new ExecutionCallback();

        ASSERT_NE(nullptr, executionCallback.get());

        Return<ErrorStatus> executionLaunchStatus =

                preparedModel->execute(request, executionCallback);

    Return<ErrorStatus> executionLaunchStatus = preparedModel->execute(request, executionCallback);

    ASSERT_TRUE(executionLaunchStatus.isOk());

    EXPECT_EQ(ErrorStatus::NONE, static_cast<ErrorStatus>(executionLaunchStatus));



        // retrieve execution status

    // Retrieve execution status.

    executionCallback->wait();

    ASSERT_EQ(ErrorStatus::NONE, executionCallback->getStatus());



        // validate results

        outputMemory->read();

        copy_back(&test, outputs_info, outputPtr);

        outputMemory->commit();

        // Filter out don't cares

        MixedTyped filtered_golden = filter(golden, is_ignored);

        MixedTyped filtered_test = filter(test, is_ignored);

    // Retrieve execution results.

    const std::vector<TestBuffer> outputs = getOutputBuffers(request);



        // We want "close-enough" results for float

        compare(filtered_golden, filtered_test, fpAtol, fpRtol);

    }

    // We want "close-enough" results.

    checkResults(testModel, outputs);

}



void Execute(const sp<IDevice>& device, std::function<Model(void)> create_model,

             std::function<bool(int)> is_ignored, const std::vector<MixedTypedExample>& examples) {

    Model model = create_model();

void Execute(const sp<IDevice>& device, const TestModel& testModel) {

    Model model = createModel(testModel);



    // see if service can handle model

    bool fullySupportsModel = false;
@@ -199,7 +173,6 @@ void Execute(const sp<IDevice>& device, std::function<Model(void)> create_model, 


    // launch prepare model

    sp<PreparedModelCallback> preparedModelCallback = new PreparedModelCallback();

    ASSERT_NE(nullptr, preparedModelCallback.get());

    Return<ErrorStatus> prepareLaunchStatus = device->prepareModel_1_1(

            model, ExecutionPreference::FAST_SINGLE_ANSWER, preparedModelCallback);

    ASSERT_TRUE(prepareLaunchStatus.isOk());
@@ -223,8 +196,7 @@ void Execute(const sp<IDevice>& device, std::function<Model(void)> create_model, 
    EXPECT_EQ(ErrorStatus::NONE, prepareReturnStatus);

    ASSERT_NE(nullptr, preparedModel.get());



    EvaluatePreparedModel(preparedModel, is_ignored, examples,

                          model.relaxComputationFloat32toFloat16, 1e-5f, 1e-5f);

    EvaluatePreparedModel(preparedModel, testModel);

}



}  // namespace generated_tests

neuralnetworks/1.1/vts/functional/GeneratedTestHarness.h

+3 −6
Original line number Diff line number Diff line
@@ -18,9 +18,6 @@ 
#define ANDROID_HARDWARE_NEURALNETWORKS_V1_1_GENERATED_TEST_HARNESS_H



#include <android/hardware/neuralnetworks/1.1/IDevice.h>

#include <android/hardware/neuralnetworks/1.1/types.h>

#include <functional>

#include <vector>

#include "TestHarness.h"



namespace android {
@@ -29,9 +26,9 @@ namespace neuralnetworks { 
namespace V1_1 {

namespace generated_tests {



void Execute(const sp<V1_1::IDevice>& device, std::function<V1_1::Model(void)> create_model,

             std::function<bool(int)> is_ignored,

             const std::vector<::test_helper::MixedTypedExample>& examples);

Model createModel(const ::test_helper::TestModel& testModel);



void Execute(const sp<V1_1::IDevice>& device, const ::test_helper::TestModel& testModel);



}  // namespace generated_tests

}  // namespace V1_1

neuralnetworks/1.1/vts/functional/GeneratedTests.h

+1 −10
Original line number Diff line number Diff line
@@ -14,20 +14,11 @@ 
 * limitations under the License.

 */



#include <android/hidl/memory/1.0/IMemory.h>

#include <hidlmemory/mapping.h>



#include "1.0/Utils.h"

#include "GeneratedTestHarness.h"

#include "MemoryUtils.h"

#include "TestHarness.h"

#include "VtsHalNeuralnetworks.h"



namespace android::hardware::neuralnetworks::V1_1::vts::functional {



std::vector<Request> createRequests(const std::vector<::test_helper::MixedTypedExample>& examples);



}  // namespace android::hardware::neuralnetworks::V1_1::vts::functional



namespace android::hardware::neuralnetworks::V1_1::generated_tests {



using namespace android::hardware::neuralnetworks::V1_1::vts::functional;

neuralnetworks/1.1/vts/functional/ValidateRequest.cpp

+4 −108
Original line number Diff line number Diff line
@@ -16,14 +16,8 @@ 


#define LOG_TAG "neuralnetworks_hidl_hal_test"



#include <android-base/logging.h>

#include <android/hidl/memory/1.0/IMemory.h>

#include <hidlmemory/mapping.h>



#include "1.0/Callbacks.h"

#include "1.0/Utils.h"

#include "MemoryUtils.h"

#include "TestHarness.h"

#include "VtsHalNeuralnetworks.h"



namespace android {
@@ -35,13 +29,8 @@ namespace functional { 


using ::android::hardware::neuralnetworks::V1_0::ErrorStatus;

using ::android::hardware::neuralnetworks::V1_0::Request;

using ::android::hardware::neuralnetworks::V1_0::RequestArgument;

using ::android::hardware::neuralnetworks::V1_0::implementation::ExecutionCallback;

using ::android::hardware::neuralnetworks::V1_1::IPreparedModel;

using ::android::hidl::memory::V1_0::IMemory;

using ::test_helper::for_all;

using ::test_helper::MixedTyped;

using ::test_helper::MixedTypedExample;



///////////////////////// UTILITY FUNCTIONS /////////////////////////
@@ -87,104 +76,11 @@ static void removeOutputTest(const sp<IPreparedModel>& preparedModel, const Requ 


///////////////////////////// ENTRY POINT //////////////////////////////////



std::vector<Request> createRequests(const std::vector<MixedTypedExample>& examples) {

    const uint32_t INPUT = 0;

    const uint32_t OUTPUT = 1;



    std::vector<Request> requests;



    for (auto& example : examples) {

        const MixedTyped& inputs = example.operands.first;

        const MixedTyped& outputs = example.operands.second;



        std::vector<RequestArgument> inputs_info, outputs_info;

        uint32_t inputSize = 0, outputSize = 0;



        // This function only partially specifies the metadata (vector of RequestArguments).

        // The contents are copied over below.

        for_all(inputs, [&inputs_info, &inputSize](int index, auto, auto s) {

            if (inputs_info.size() <= static_cast<size_t>(index)) inputs_info.resize(index + 1);

            RequestArgument arg = {

                    .location = {.poolIndex = INPUT,

                                 .offset = 0,

                                 .length = static_cast<uint32_t>(s)},

                    .dimensions = {},

            };

            RequestArgument arg_empty = {

                    .hasNoValue = true,

            };

            inputs_info[index] = s ? arg : arg_empty;

            inputSize += s;

        });

        // Compute offset for inputs 1 and so on

        {

            size_t offset = 0;

            for (auto& i : inputs_info) {

                if (!i.hasNoValue) i.location.offset = offset;

                offset += i.location.length;

            }

        }



        // Go through all outputs, initialize RequestArgument descriptors

        for_all(outputs, [&outputs_info, &outputSize](int index, auto, auto s) {

            if (outputs_info.size() <= static_cast<size_t>(index)) outputs_info.resize(index + 1);

            RequestArgument arg = {

                    .location = {.poolIndex = OUTPUT,

                                 .offset = 0,

                                 .length = static_cast<uint32_t>(s)},

                    .dimensions = {},

            };

            outputs_info[index] = arg;

            outputSize += s;

        });

        // Compute offset for outputs 1 and so on

        {

            size_t offset = 0;

            for (auto& i : outputs_info) {

                i.location.offset = offset;

                offset += i.location.length;

            }

        }

        std::vector<hidl_memory> pools = {nn::allocateSharedMemory(inputSize),

                                          nn::allocateSharedMemory(outputSize)};

        if (pools[INPUT].size() == 0 || pools[OUTPUT].size() == 0) {

            return {};

        }



        // map pool

        sp<IMemory> inputMemory = mapMemory(pools[INPUT]);

        if (inputMemory == nullptr) {

            return {};

        }

        char* inputPtr = reinterpret_cast<char*>(static_cast<void*>(inputMemory->getPointer()));

        if (inputPtr == nullptr) {

            return {};

        }



        // initialize pool

        inputMemory->update();

        for_all(inputs, [&inputs_info, inputPtr](int index, auto p, auto s) {

            char* begin = (char*)p;

            char* end = begin + s;

            // TODO: handle more than one input

            std::copy(begin, end, inputPtr + inputs_info[index].location.offset);

        });

        inputMemory->commit();



        requests.push_back({.inputs = inputs_info, .outputs = outputs_info, .pools = pools});

    }



    return requests;

}



void ValidationTest::validateRequests(const sp<IPreparedModel>& preparedModel,

                                      const std::vector<Request>& requests) {

    // validate each request

    for (const Request& request : requests) {

void ValidationTest::validateRequest(const sp<IPreparedModel>& preparedModel,

                                     const Request& request) {

    removeInputTest(preparedModel, request);

    removeOutputTest(preparedModel, request);

}

}



}  // namespace functional

}  // namespace vts