Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 8e8b70c4 authored by Xusong Wang's avatar Xusong Wang
Browse files

Modify 1.0 VTS tests to consume test struct directly. 

Implement converter utilities constructing HIDL model and request from
TestModel.

Bug: 123092187
Bug: 138718240
Test: All VTS
Change-Id: I0b26b7f41d31d5e63ed083ab5f6f269a3620f034
parent 1c4b95fb

neuralnetworks/1.0/vts/functional/Android.bp

+2 −4
Original line number Diff line number Diff line
@@ -32,12 +32,11 @@ cc_library_static { 
        "android.hidl.memory@1.0",

        "libgmock",

        "libhidlmemory",

        "libneuralnetworks_generated_test_harness",

        "libneuralnetworks_utils",

    ],

    header_libs: [

        "libneuralnetworks_headers",

        "libneuralnetworks_generated_test_harness_headers",

        "libneuralnetworks_generated_tests",

    ],

}
@@ -60,13 +59,12 @@ cc_defaults { 
        "android.hidl.memory@1.0",

        "libgmock",

        "libhidlmemory",

        "libneuralnetworks_generated_test_harness",

        "libneuralnetworks_utils",

        "VtsHalNeuralNetworksV1_0_utils",

    ],

    header_libs: [

        "libneuralnetworks_headers",

        "libneuralnetworks_generated_test_harness_headers",

        "libneuralnetworks_generated_tests",

    ],

    test_suites: ["general-tests"],

}

neuralnetworks/1.0/vts/functional/GeneratedTestHarness.cpp

+100 −125
Original line number Diff line number Diff line
@@ -15,6 +15,7 @@ 
 */



#include "GeneratedTestHarness.h"



#include "1.0/Callbacks.h"

#include "1.0/Utils.h"

#include "MemoryUtils.h"
@@ -28,6 +29,7 @@ 
#include <android/hidl/memory/1.0/IMemory.h>

#include <hidlmemory/mapping.h>



#include <gtest/gtest.h>

#include <iostream>



namespace android {
@@ -36,6 +38,7 @@ namespace neuralnetworks { 
namespace V1_0 {

namespace generated_tests {



using namespace test_helper;

using ::android::hardware::neuralnetworks::V1_0::ErrorStatus;

using ::android::hardware::neuralnetworks::V1_0::IDevice;

using ::android::hardware::neuralnetworks::V1_0::IPreparedModel;
@@ -45,137 +48,111 @@ using ::android::hardware::neuralnetworks::V1_0::RequestArgument; 
using ::android::hardware::neuralnetworks::V1_0::implementation::ExecutionCallback;

using ::android::hardware::neuralnetworks::V1_0::implementation::PreparedModelCallback;

using ::android::hidl::memory::V1_0::IMemory;

using ::test_helper::compare;

using ::test_helper::filter;

using ::test_helper::for_all;

using ::test_helper::MixedTyped;

using ::test_helper::MixedTypedExample;

using ::test_helper::resize_accordingly;



// Top level driver for models and examples generated by test_generator.py

// Test driver for those generated from ml/nn/runtime/test/spec

void EvaluatePreparedModel(sp<IPreparedModel>& preparedModel, std::function<bool(int)> is_ignored,

                           const std::vector<MixedTypedExample>& examples, float fpAtol,

                           float fpRtol) {

    const uint32_t INPUT = 0;

    const uint32_t OUTPUT = 1;



    int example_no = 1;

    for (auto& example : examples) {

        SCOPED_TRACE(example_no++);

        const MixedTyped& inputs = example.operands.first;

        const MixedTyped& golden = example.operands.second;



        CHECK(inputs.float16Operands.empty()) << "float16 is not supported in 1.0";



        std::vector<RequestArgument> inputs_info, outputs_info;

        uint32_t inputSize = 0, outputSize = 0;

        // This function only partially specifies the metadata (vector of RequestArguments).

        // The contents are copied over below.

        for_all(inputs, [&inputs_info, &inputSize](int index, auto, auto s) {

            if (inputs_info.size() <= static_cast<size_t>(index)) inputs_info.resize(index + 1);

            RequestArgument arg = {

                    .location = {.poolIndex = INPUT,

                                 .offset = 0,

                                 .length = static_cast<uint32_t>(s)},

                    .dimensions = {},

            };

            RequestArgument arg_empty = {

                    .hasNoValue = true,

            };

            inputs_info[index] = s ? arg : arg_empty;

            inputSize += s;

        });

        // Compute offset for inputs 1 and so on

        {

            size_t offset = 0;

            for (auto& i : inputs_info) {

                if (!i.hasNoValue) i.location.offset = offset;

                offset += i.location.length;

Model createModel(const TestModel& testModel) {

    // Model operands.

    hidl_vec<Operand> operands(testModel.operands.size());

    size_t constCopySize = 0, constRefSize = 0;

    for (uint32_t i = 0; i < testModel.operands.size(); i++) {

        const auto& op = testModel.operands[i];



        DataLocation loc = {};

        if (op.lifetime == TestOperandLifeTime::CONSTANT_COPY) {

            loc = {.poolIndex = 0,

                   .offset = static_cast<uint32_t>(constCopySize),

                   .length = static_cast<uint32_t>(op.data.size())};

            constCopySize += op.data.alignedSize();

        } else if (op.lifetime == TestOperandLifeTime::CONSTANT_REFERENCE) {

            loc = {.poolIndex = 0,

                   .offset = static_cast<uint32_t>(constRefSize),

                   .length = static_cast<uint32_t>(op.data.size())};

            constRefSize += op.data.alignedSize();

        }



        operands[i] = {.type = static_cast<OperandType>(op.type),

                       .dimensions = op.dimensions,

                       .numberOfConsumers = op.numberOfConsumers,

                       .scale = op.scale,

                       .zeroPoint = op.zeroPoint,

                       .lifetime = static_cast<OperandLifeTime>(op.lifetime),

                       .location = loc};

    }



        MixedTyped test;  // holding test results



        // Go through all outputs, initialize RequestArgument descriptors

        resize_accordingly(golden, test);

        for_all(golden, [&outputs_info, &outputSize](int index, auto, auto s) {

            if (outputs_info.size() <= static_cast<size_t>(index)) outputs_info.resize(index + 1);

            RequestArgument arg = {

                    .location = {.poolIndex = OUTPUT,

                                 .offset = 0,

                                 .length = static_cast<uint32_t>(s)},

                    .dimensions = {},

            };

            outputs_info[index] = arg;

            outputSize += s;

    // Model operations.

    hidl_vec<Operation> operations(testModel.operations.size());

    std::transform(testModel.operations.begin(), testModel.operations.end(), operations.begin(),

                   [](const TestOperation& op) -> Operation {

                       return {.type = static_cast<OperationType>(op.type),

                               .inputs = op.inputs,

                               .outputs = op.outputs};

                   });

        // Compute offset for outputs 1 and so on

        {

            size_t offset = 0;

            for (auto& i : outputs_info) {

                i.location.offset = offset;

                offset += i.location.length;



    // Constant copies.

    hidl_vec<uint8_t> operandValues(constCopySize);

    for (uint32_t i = 0; i < testModel.operands.size(); i++) {

        const auto& op = testModel.operands[i];

        if (op.lifetime == TestOperandLifeTime::CONSTANT_COPY) {

            const uint8_t* begin = op.data.get<uint8_t>();

            const uint8_t* end = begin + op.data.size();

            std::copy(begin, end, operandValues.data() + operands[i].location.offset);

        }

    }

        std::vector<hidl_memory> pools = {nn::allocateSharedMemory(inputSize),

                                          nn::allocateSharedMemory(outputSize)};

        ASSERT_NE(0ull, pools[INPUT].size());

        ASSERT_NE(0ull, pools[OUTPUT].size());



    // Shared memory.

    hidl_vec<hidl_memory> pools;

    if (constRefSize > 0) {

        hidl_vec_push_back(&pools, nn::allocateSharedMemory(constRefSize));

        CHECK_NE(pools[0].size(), 0u);



        // load data

        sp<IMemory> inputMemory = mapMemory(pools[INPUT]);

        sp<IMemory> outputMemory = mapMemory(pools[OUTPUT]);

        ASSERT_NE(nullptr, inputMemory.get());

        ASSERT_NE(nullptr, outputMemory.get());

        char* inputPtr = reinterpret_cast<char*>(static_cast<void*>(inputMemory->getPointer()));

        char* outputPtr = reinterpret_cast<char*>(static_cast<void*>(outputMemory->getPointer()));

        ASSERT_NE(nullptr, inputPtr);

        ASSERT_NE(nullptr, outputPtr);

        inputMemory->update();

        outputMemory->update();



        // Go through all inputs, copy the values

        for_all(inputs, [&inputs_info, inputPtr](int index, auto p, auto s) {

            char* begin = (char*)p;

            char* end = begin + s;

            // TODO: handle more than one input

            std::copy(begin, end, inputPtr + inputs_info[index].location.offset);

        });

        sp<IMemory> mappedMemory = mapMemory(pools[0]);

        CHECK(mappedMemory.get() != nullptr);

        uint8_t* mappedPtr =

                reinterpret_cast<uint8_t*>(static_cast<void*>(mappedMemory->getPointer()));

        CHECK(mappedPtr != nullptr);



        for (uint32_t i = 0; i < testModel.operands.size(); i++) {

            const auto& op = testModel.operands[i];

            if (op.lifetime == TestOperandLifeTime::CONSTANT_REFERENCE) {

                const uint8_t* begin = op.data.get<uint8_t>();

                const uint8_t* end = begin + op.data.size();

                std::copy(begin, end, mappedPtr + operands[i].location.offset);

            }

        }

    }



        inputMemory->commit();

        outputMemory->commit();

    return {.operands = std::move(operands),

            .operations = std::move(operations),

            .inputIndexes = testModel.inputIndexes,

            .outputIndexes = testModel.outputIndexes,

            .operandValues = std::move(operandValues),

            .pools = std::move(pools)};

}



        const Request request = {.inputs = inputs_info, .outputs = outputs_info, .pools = pools};

// Top level driver for models and examples generated by test_generator.py

// Test driver for those generated from ml/nn/runtime/test/spec

void EvaluatePreparedModel(const sp<IPreparedModel>& preparedModel, const TestModel& testModel) {

    const Request request = createRequest(testModel);



        // launch execution

    // Launch execution.

    sp<ExecutionCallback> executionCallback = new ExecutionCallback();

        ASSERT_NE(nullptr, executionCallback.get());

        Return<ErrorStatus> executionLaunchStatus =

                preparedModel->execute(request, executionCallback);

    Return<ErrorStatus> executionLaunchStatus = preparedModel->execute(request, executionCallback);

    ASSERT_TRUE(executionLaunchStatus.isOk());

    EXPECT_EQ(ErrorStatus::NONE, static_cast<ErrorStatus>(executionLaunchStatus));



        // retrieve execution status

    // Retrieve execution status.

    executionCallback->wait();

    ASSERT_EQ(ErrorStatus::NONE, executionCallback->getStatus());



        // validate results

        outputMemory->read();

        copy_back(&test, outputs_info, outputPtr);

        outputMemory->commit();

        // Filter out don't cares

        MixedTyped filtered_golden = filter(golden, is_ignored);

        MixedTyped filtered_test = filter(test, is_ignored);

    // Retrieve execution results.

    const std::vector<TestBuffer> outputs = getOutputBuffers(request);



        // We want "close-enough" results for float

        compare(filtered_golden, filtered_test, fpAtol, fpRtol);

    }

    // We want "close-enough" results.

    checkResults(testModel, outputs);

}



void Execute(const sp<IDevice>& device, std::function<Model(void)> create_model,

             std::function<bool(int)> is_ignored, const std::vector<MixedTypedExample>& examples) {

    Model model = create_model();

void Execute(const sp<IDevice>& device, const TestModel& testModel) {

    Model model = createModel(testModel);



    // see if service can handle model

    bool fullySupportsModel = false;
@@ -190,7 +167,6 @@ void Execute(const sp<IDevice>& device, std::function<Model(void)> create_model, 


    // launch prepare model

    sp<PreparedModelCallback> preparedModelCallback = new PreparedModelCallback();

    ASSERT_NE(nullptr, preparedModelCallback.get());

    Return<ErrorStatus> prepareLaunchStatus = device->prepareModel(model, preparedModelCallback);

    ASSERT_TRUE(prepareLaunchStatus.isOk());

    ASSERT_EQ(ErrorStatus::NONE, static_cast<ErrorStatus>(prepareLaunchStatus));
@@ -213,8 +189,7 @@ void Execute(const sp<IDevice>& device, std::function<Model(void)> create_model, 
    EXPECT_EQ(ErrorStatus::NONE, prepareReturnStatus);

    ASSERT_NE(nullptr, preparedModel.get());



    float fpAtol = 1e-5f, fpRtol = 5.0f * 1.1920928955078125e-7f;

    EvaluatePreparedModel(preparedModel, is_ignored, examples, fpAtol, fpRtol);

    EvaluatePreparedModel(preparedModel, testModel);

}



}  // namespace generated_tests

neuralnetworks/1.0/vts/functional/GeneratedTestHarness.h

+2 −3
Original line number Diff line number Diff line
@@ -26,10 +26,9 @@ namespace neuralnetworks { 
namespace V1_0 {

namespace generated_tests {



using ::test_helper::MixedTypedExample;

Model createModel(const ::test_helper::TestModel& testModel);



void Execute(const sp<V1_0::IDevice>& device, std::function<V1_0::Model(void)> create_model,

             std::function<bool(int)> is_ignored, const std::vector<MixedTypedExample>& examples);

void Execute(const sp<V1_0::IDevice>& device, const ::test_helper::TestModel& testModel);



}  // namespace generated_tests

}  // namespace V1_0

neuralnetworks/1.0/vts/functional/GeneratedTests.h

+1 −10
Original line number Diff line number Diff line
@@ -14,20 +14,11 @@ 
 * limitations under the License.

 */



#include <android/hidl/memory/1.0/IMemory.h>

#include <hidlmemory/mapping.h>



#include "1.0/Utils.h"

#include "GeneratedTestHarness.h"

#include "MemoryUtils.h"

#include "TestHarness.h"

#include "VtsHalNeuralnetworks.h"



namespace android::hardware::neuralnetworks::V1_0::vts::functional {



std::vector<Request> createRequests(const std::vector<::test_helper::MixedTypedExample>& examples);



}  // namespace android::hardware::neuralnetworks::V1_0::vts::functional



namespace android::hardware::neuralnetworks::V1_0::generated_tests {



using namespace android::hardware::neuralnetworks::V1_0::vts::functional;

neuralnetworks/1.0/vts/functional/Utils.cpp

+80 −26
Original line number Diff line number Diff line
@@ -14,45 +14,99 @@ 
 * limitations under the License.

 */



#include "GeneratedTestHarness.h"

#include "1.0/Utils.h"



#include "MemoryUtils.h"

#include "TestHarness.h"



#include <android-base/logging.h>

#include <android/hardware/neuralnetworks/1.0/types.h>

#include <android/hidl/allocator/1.0/IAllocator.h>

#include <android/hidl/memory/1.0/IMemory.h>

#include <hidlmemory/mapping.h>



#include <cstring>

#include <map>

#include <vector>



namespace android {

namespace hardware {

namespace neuralnetworks {



using namespace test_helper;

using ::android::hardware::neuralnetworks::V1_0::DataLocation;

using ::android::hardware::neuralnetworks::V1_0::Request;

using ::android::hardware::neuralnetworks::V1_0::RequestArgument;

using ::test_helper::for_each;

using ::test_helper::MixedTyped;



template <typename T>

void copy_back_(std::map<int, std::vector<T>>* dst, const std::vector<RequestArgument>& ra,

                char* src) {

    for_each<T>(*dst, [&ra, src](int index, std::vector<T>& m) {

        ASSERT_EQ(m.size(), ra[index].location.length / sizeof(T));

        char* begin = src + ra[index].location.offset;

        memcpy(m.data(), begin, ra[index].location.length);

    });

}



void copy_back(MixedTyped* dst, const std::vector<RequestArgument>& ra, char* src) {

    copy_back_(&dst->float32Operands, ra, src);

    copy_back_(&dst->int32Operands, ra, src);

    copy_back_(&dst->quant8AsymmOperands, ra, src);

    copy_back_(&dst->quant16SymmOperands, ra, src);

    copy_back_(&dst->float16Operands, ra, src);

    copy_back_(&dst->bool8Operands, ra, src);

    copy_back_(&dst->quant8ChannelOperands, ra, src);

    copy_back_(&dst->quant16AsymmOperands, ra, src);

    copy_back_(&dst->quant8SymmOperands, ra, src);

    static_assert(9 == MixedTyped::kNumTypes,

                  "Number of types in MixedTyped changed, but copy_back function wasn't updated");

using ::android::hidl::memory::V1_0::IMemory;



constexpr uint32_t kInputPoolIndex = 0;

constexpr uint32_t kOutputPoolIndex = 1;



Request createRequest(const TestModel& testModel) {

    // Model inputs.

    hidl_vec<RequestArgument> inputs(testModel.inputIndexes.size());

    size_t inputSize = 0;

    for (uint32_t i = 0; i < testModel.inputIndexes.size(); i++) {

        const auto& op = testModel.operands[testModel.inputIndexes[i]];

        if (op.data.size() == 0) {

            // Omitted input.

            inputs[i] = {.hasNoValue = true};

        } else {

            DataLocation loc = {.poolIndex = kInputPoolIndex,

                                .offset = static_cast<uint32_t>(inputSize),

                                .length = static_cast<uint32_t>(op.data.size())};

            inputSize += op.data.alignedSize();

            inputs[i] = {.hasNoValue = false, .location = loc, .dimensions = {}};

        }

    }



    // Model outputs.

    hidl_vec<RequestArgument> outputs(testModel.outputIndexes.size());

    size_t outputSize = 0;

    for (uint32_t i = 0; i < testModel.outputIndexes.size(); i++) {

        const auto& op = testModel.operands[testModel.outputIndexes[i]];

        size_t dataSize = op.data.size();

        DataLocation loc = {.poolIndex = kOutputPoolIndex,

                            .offset = static_cast<uint32_t>(outputSize),

                            .length = static_cast<uint32_t>(dataSize)};

        outputSize += op.data.alignedSize();

        outputs[i] = {.hasNoValue = false, .location = loc, .dimensions = {}};

    }



    // Allocate memory pools.

    hidl_vec<hidl_memory> pools = {nn::allocateSharedMemory(inputSize),

                                   nn::allocateSharedMemory(outputSize)};

    CHECK_NE(pools[kInputPoolIndex].size(), 0u);

    CHECK_NE(pools[kOutputPoolIndex].size(), 0u);

    sp<IMemory> inputMemory = mapMemory(pools[kInputPoolIndex]);

    CHECK(inputMemory.get() != nullptr);

    uint8_t* inputPtr = static_cast<uint8_t*>(static_cast<void*>(inputMemory->getPointer()));

    CHECK(inputPtr != nullptr);



    // Copy input data to the memory pool.

    for (uint32_t i = 0; i < testModel.inputIndexes.size(); i++) {

        const auto& op = testModel.operands[testModel.inputIndexes[i]];

        if (op.data.size() > 0) {

            const uint8_t* begin = op.data.get<uint8_t>();

            const uint8_t* end = begin + op.data.size();

            std::copy(begin, end, inputPtr + inputs[i].location.offset);

        }

    }



    return {.inputs = std::move(inputs), .outputs = std::move(outputs), .pools = std::move(pools)};

}



std::vector<TestBuffer> getOutputBuffers(const Request& request) {

    sp<IMemory> outputMemory = mapMemory(request.pools[kOutputPoolIndex]);

    CHECK(outputMemory.get() != nullptr);

    uint8_t* outputPtr = static_cast<uint8_t*>(static_cast<void*>(outputMemory->getPointer()));

    CHECK(outputPtr != nullptr);



    // Copy out output results.

    std::vector<TestBuffer> outputBuffers;

    for (const auto& output : request.outputs) {

        outputBuffers.emplace_back(output.location.length, outputPtr + output.location.offset);

    }



    return outputBuffers;

}



}  // namespace neuralnetworks