Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 2afe3aca authored by Xusong Wang's avatar Xusong Wang Committed by Android (Google) Code Review
Browse files

Merge changes from topic "nn_unify_test_struct" 

* changes:
  Modify validateBurstSanitized to take a single request.
  Modify 1.2 VTS tests to consume test struct directly.
  Modify 1.1 VTS tests to consume test struct directly.
  Modify 1.0 VTS tests to consume test struct directly.
parents 6afc1740 8fc4622b

neuralnetworks/1.0/vts/functional/Android.bp

+2 −4
Original line number Diff line number Diff line
@@ -32,12 +32,11 @@ cc_library_static { 
        "android.hidl.memory@1.0",

        "libgmock",

        "libhidlmemory",

        "libneuralnetworks_generated_test_harness",

        "libneuralnetworks_utils",

    ],

    header_libs: [

        "libneuralnetworks_headers",

        "libneuralnetworks_generated_test_harness_headers",

        "libneuralnetworks_generated_tests",

    ],

}
@@ -60,13 +59,12 @@ cc_defaults { 
        "android.hidl.memory@1.0",

        "libgmock",

        "libhidlmemory",

        "libneuralnetworks_generated_test_harness",

        "libneuralnetworks_utils",

        "VtsHalNeuralNetworksV1_0_utils",

    ],

    header_libs: [

        "libneuralnetworks_headers",

        "libneuralnetworks_generated_test_harness_headers",

        "libneuralnetworks_generated_tests",

    ],

    test_suites: ["general-tests"],

}

neuralnetworks/1.0/vts/functional/GeneratedTestHarness.cpp

+100 −125
Original line number Diff line number Diff line
@@ -15,6 +15,7 @@ 
 */



#include "GeneratedTestHarness.h"



#include "1.0/Callbacks.h"

#include "1.0/Utils.h"

#include "MemoryUtils.h"
@@ -28,6 +29,7 @@ 
#include <android/hidl/memory/1.0/IMemory.h>

#include <hidlmemory/mapping.h>



#include <gtest/gtest.h>

#include <iostream>



namespace android {
@@ -36,6 +38,7 @@ namespace neuralnetworks { 
namespace V1_0 {

namespace generated_tests {



using namespace test_helper;

using ::android::hardware::neuralnetworks::V1_0::ErrorStatus;

using ::android::hardware::neuralnetworks::V1_0::IDevice;

using ::android::hardware::neuralnetworks::V1_0::IPreparedModel;
@@ -45,137 +48,111 @@ using ::android::hardware::neuralnetworks::V1_0::RequestArgument; 
using ::android::hardware::neuralnetworks::V1_0::implementation::ExecutionCallback;

using ::android::hardware::neuralnetworks::V1_0::implementation::PreparedModelCallback;

using ::android::hidl::memory::V1_0::IMemory;

using ::test_helper::compare;

using ::test_helper::filter;

using ::test_helper::for_all;

using ::test_helper::MixedTyped;

using ::test_helper::MixedTypedExample;

using ::test_helper::resize_accordingly;



// Top level driver for models and examples generated by test_generator.py

// Test driver for those generated from ml/nn/runtime/test/spec

void EvaluatePreparedModel(sp<IPreparedModel>& preparedModel, std::function<bool(int)> is_ignored,

                           const std::vector<MixedTypedExample>& examples, float fpAtol,

                           float fpRtol) {

    const uint32_t INPUT = 0;

    const uint32_t OUTPUT = 1;



    int example_no = 1;

    for (auto& example : examples) {

        SCOPED_TRACE(example_no++);

        const MixedTyped& inputs = example.operands.first;

        const MixedTyped& golden = example.operands.second;



        CHECK(inputs.float16Operands.empty()) << "float16 is not supported in 1.0";



        std::vector<RequestArgument> inputs_info, outputs_info;

        uint32_t inputSize = 0, outputSize = 0;

        // This function only partially specifies the metadata (vector of RequestArguments).

        // The contents are copied over below.

        for_all(inputs, [&inputs_info, &inputSize](int index, auto, auto s) {

            if (inputs_info.size() <= static_cast<size_t>(index)) inputs_info.resize(index + 1);

            RequestArgument arg = {

                    .location = {.poolIndex = INPUT,

                                 .offset = 0,

                                 .length = static_cast<uint32_t>(s)},

                    .dimensions = {},

            };

            RequestArgument arg_empty = {

                    .hasNoValue = true,

            };

            inputs_info[index] = s ? arg : arg_empty;

            inputSize += s;

        });

        // Compute offset for inputs 1 and so on

        {

            size_t offset = 0;

            for (auto& i : inputs_info) {

                if (!i.hasNoValue) i.location.offset = offset;

                offset += i.location.length;

Model createModel(const TestModel& testModel) {

    // Model operands.

    hidl_vec<Operand> operands(testModel.operands.size());

    size_t constCopySize = 0, constRefSize = 0;

    for (uint32_t i = 0; i < testModel.operands.size(); i++) {

        const auto& op = testModel.operands[i];



        DataLocation loc = {};

        if (op.lifetime == TestOperandLifeTime::CONSTANT_COPY) {

            loc = {.poolIndex = 0,

                   .offset = static_cast<uint32_t>(constCopySize),

                   .length = static_cast<uint32_t>(op.data.size())};

            constCopySize += op.data.alignedSize();

        } else if (op.lifetime == TestOperandLifeTime::CONSTANT_REFERENCE) {

            loc = {.poolIndex = 0,

                   .offset = static_cast<uint32_t>(constRefSize),

                   .length = static_cast<uint32_t>(op.data.size())};

            constRefSize += op.data.alignedSize();

        }



        operands[i] = {.type = static_cast<OperandType>(op.type),

                       .dimensions = op.dimensions,

                       .numberOfConsumers = op.numberOfConsumers,

                       .scale = op.scale,

                       .zeroPoint = op.zeroPoint,

                       .lifetime = static_cast<OperandLifeTime>(op.lifetime),

                       .location = loc};

    }



        MixedTyped test;  // holding test results



        // Go through all outputs, initialize RequestArgument descriptors

        resize_accordingly(golden, test);

        for_all(golden, [&outputs_info, &outputSize](int index, auto, auto s) {

            if (outputs_info.size() <= static_cast<size_t>(index)) outputs_info.resize(index + 1);

            RequestArgument arg = {

                    .location = {.poolIndex = OUTPUT,

                                 .offset = 0,

                                 .length = static_cast<uint32_t>(s)},

                    .dimensions = {},

            };

            outputs_info[index] = arg;

            outputSize += s;

    // Model operations.

    hidl_vec<Operation> operations(testModel.operations.size());

    std::transform(testModel.operations.begin(), testModel.operations.end(), operations.begin(),

                   [](const TestOperation& op) -> Operation {

                       return {.type = static_cast<OperationType>(op.type),

                               .inputs = op.inputs,

                               .outputs = op.outputs};

                   });

        // Compute offset for outputs 1 and so on

        {

            size_t offset = 0;

            for (auto& i : outputs_info) {

                i.location.offset = offset;

                offset += i.location.length;



    // Constant copies.

    hidl_vec<uint8_t> operandValues(constCopySize);

    for (uint32_t i = 0; i < testModel.operands.size(); i++) {

        const auto& op = testModel.operands[i];

        if (op.lifetime == TestOperandLifeTime::CONSTANT_COPY) {

            const uint8_t* begin = op.data.get<uint8_t>();

            const uint8_t* end = begin + op.data.size();

            std::copy(begin, end, operandValues.data() + operands[i].location.offset);

        }

    }

        std::vector<hidl_memory> pools = {nn::allocateSharedMemory(inputSize),

                                          nn::allocateSharedMemory(outputSize)};

        ASSERT_NE(0ull, pools[INPUT].size());

        ASSERT_NE(0ull, pools[OUTPUT].size());



    // Shared memory.

    hidl_vec<hidl_memory> pools;

    if (constRefSize > 0) {

        hidl_vec_push_back(&pools, nn::allocateSharedMemory(constRefSize));

        CHECK_NE(pools[0].size(), 0u);



        // load data

        sp<IMemory> inputMemory = mapMemory(pools[INPUT]);

        sp<IMemory> outputMemory = mapMemory(pools[OUTPUT]);

        ASSERT_NE(nullptr, inputMemory.get());

        ASSERT_NE(nullptr, outputMemory.get());

        char* inputPtr = reinterpret_cast<char*>(static_cast<void*>(inputMemory->getPointer()));

        char* outputPtr = reinterpret_cast<char*>(static_cast<void*>(outputMemory->getPointer()));

        ASSERT_NE(nullptr, inputPtr);

        ASSERT_NE(nullptr, outputPtr);

        inputMemory->update();

        outputMemory->update();



        // Go through all inputs, copy the values

        for_all(inputs, [&inputs_info, inputPtr](int index, auto p, auto s) {

            char* begin = (char*)p;

            char* end = begin + s;

            // TODO: handle more than one input

            std::copy(begin, end, inputPtr + inputs_info[index].location.offset);

        });

        sp<IMemory> mappedMemory = mapMemory(pools[0]);

        CHECK(mappedMemory.get() != nullptr);

        uint8_t* mappedPtr =

                reinterpret_cast<uint8_t*>(static_cast<void*>(mappedMemory->getPointer()));

        CHECK(mappedPtr != nullptr);



        for (uint32_t i = 0; i < testModel.operands.size(); i++) {

            const auto& op = testModel.operands[i];

            if (op.lifetime == TestOperandLifeTime::CONSTANT_REFERENCE) {

                const uint8_t* begin = op.data.get<uint8_t>();

                const uint8_t* end = begin + op.data.size();

                std::copy(begin, end, mappedPtr + operands[i].location.offset);

            }

        }

    }



        inputMemory->commit();

        outputMemory->commit();

    return {.operands = std::move(operands),

            .operations = std::move(operations),

            .inputIndexes = testModel.inputIndexes,

            .outputIndexes = testModel.outputIndexes,

            .operandValues = std::move(operandValues),

            .pools = std::move(pools)};

}



        const Request request = {.inputs = inputs_info, .outputs = outputs_info, .pools = pools};

// Top level driver for models and examples generated by test_generator.py

// Test driver for those generated from ml/nn/runtime/test/spec

void EvaluatePreparedModel(const sp<IPreparedModel>& preparedModel, const TestModel& testModel) {

    const Request request = createRequest(testModel);



        // launch execution

    // Launch execution.

    sp<ExecutionCallback> executionCallback = new ExecutionCallback();

        ASSERT_NE(nullptr, executionCallback.get());

        Return<ErrorStatus> executionLaunchStatus =

                preparedModel->execute(request, executionCallback);

    Return<ErrorStatus> executionLaunchStatus = preparedModel->execute(request, executionCallback);

    ASSERT_TRUE(executionLaunchStatus.isOk());

    EXPECT_EQ(ErrorStatus::NONE, static_cast<ErrorStatus>(executionLaunchStatus));



        // retrieve execution status

    // Retrieve execution status.

    executionCallback->wait();

    ASSERT_EQ(ErrorStatus::NONE, executionCallback->getStatus());



        // validate results

        outputMemory->read();

        copy_back(&test, outputs_info, outputPtr);

        outputMemory->commit();

        // Filter out don't cares

        MixedTyped filtered_golden = filter(golden, is_ignored);

        MixedTyped filtered_test = filter(test, is_ignored);

    // Retrieve execution results.

    const std::vector<TestBuffer> outputs = getOutputBuffers(request);



        // We want "close-enough" results for float

        compare(filtered_golden, filtered_test, fpAtol, fpRtol);

    }

    // We want "close-enough" results.

    checkResults(testModel, outputs);

}



void Execute(const sp<IDevice>& device, std::function<Model(void)> create_model,

             std::function<bool(int)> is_ignored, const std::vector<MixedTypedExample>& examples) {

    Model model = create_model();

void Execute(const sp<IDevice>& device, const TestModel& testModel) {

    Model model = createModel(testModel);



    // see if service can handle model

    bool fullySupportsModel = false;
@@ -190,7 +167,6 @@ void Execute(const sp<IDevice>& device, std::function<Model(void)> create_model, 


    // launch prepare model

    sp<PreparedModelCallback> preparedModelCallback = new PreparedModelCallback();

    ASSERT_NE(nullptr, preparedModelCallback.get());

    Return<ErrorStatus> prepareLaunchStatus = device->prepareModel(model, preparedModelCallback);

    ASSERT_TRUE(prepareLaunchStatus.isOk());

    ASSERT_EQ(ErrorStatus::NONE, static_cast<ErrorStatus>(prepareLaunchStatus));
@@ -213,8 +189,7 @@ void Execute(const sp<IDevice>& device, std::function<Model(void)> create_model, 
    EXPECT_EQ(ErrorStatus::NONE, prepareReturnStatus);

    ASSERT_NE(nullptr, preparedModel.get());



    float fpAtol = 1e-5f, fpRtol = 5.0f * 1.1920928955078125e-7f;

    EvaluatePreparedModel(preparedModel, is_ignored, examples, fpAtol, fpRtol);

    EvaluatePreparedModel(preparedModel, testModel);

}



}  // namespace generated_tests

neuralnetworks/1.0/vts/functional/GeneratedTestHarness.h

+2 −3
Original line number Diff line number Diff line
@@ -26,10 +26,9 @@ namespace neuralnetworks { 
namespace V1_0 {

namespace generated_tests {



using ::test_helper::MixedTypedExample;

Model createModel(const ::test_helper::TestModel& testModel);



void Execute(const sp<V1_0::IDevice>& device, std::function<V1_0::Model(void)> create_model,

             std::function<bool(int)> is_ignored, const std::vector<MixedTypedExample>& examples);

void Execute(const sp<V1_0::IDevice>& device, const ::test_helper::TestModel& testModel);



}  // namespace generated_tests

}  // namespace V1_0

neuralnetworks/1.0/vts/functional/GeneratedTests.h

+1 −10
Original line number Diff line number Diff line
@@ -14,20 +14,11 @@ 
 * limitations under the License.

 */



#include <android/hidl/memory/1.0/IMemory.h>

#include <hidlmemory/mapping.h>



#include "1.0/Utils.h"

#include "GeneratedTestHarness.h"

#include "MemoryUtils.h"

#include "TestHarness.h"

#include "VtsHalNeuralnetworks.h"



namespace android::hardware::neuralnetworks::V1_0::vts::functional {



std::vector<Request> createRequests(const std::vector<::test_helper::MixedTypedExample>& examples);



}  // namespace android::hardware::neuralnetworks::V1_0::vts::functional



namespace android::hardware::neuralnetworks::V1_0::generated_tests {



using namespace android::hardware::neuralnetworks::V1_0::vts::functional;

neuralnetworks/1.0/vts/functional/Utils.cpp

+89 −26
Original line number Diff line number Diff line
@@ -14,45 +14,108 @@ 
 * limitations under the License.

 */



#include "GeneratedTestHarness.h"

#include "1.0/Utils.h"



#include "MemoryUtils.h"

#include "TestHarness.h"



#include <android-base/logging.h>

#include <android/hardware/neuralnetworks/1.0/types.h>

#include <android/hidl/allocator/1.0/IAllocator.h>

#include <android/hidl/memory/1.0/IMemory.h>

#include <hidlmemory/mapping.h>



#include <cstring>

#include <map>

#include <algorithm>

#include <vector>



namespace android {

namespace hardware {

namespace neuralnetworks {



using namespace test_helper;

using ::android::hardware::neuralnetworks::V1_0::DataLocation;

using ::android::hardware::neuralnetworks::V1_0::Request;

using ::android::hardware::neuralnetworks::V1_0::RequestArgument;

using ::test_helper::for_each;

using ::test_helper::MixedTyped;



template <typename T>

void copy_back_(std::map<int, std::vector<T>>* dst, const std::vector<RequestArgument>& ra,

                char* src) {

    for_each<T>(*dst, [&ra, src](int index, std::vector<T>& m) {

        ASSERT_EQ(m.size(), ra[index].location.length / sizeof(T));

        char* begin = src + ra[index].location.offset;

        memcpy(m.data(), begin, ra[index].location.length);

    });

}



void copy_back(MixedTyped* dst, const std::vector<RequestArgument>& ra, char* src) {

    copy_back_(&dst->float32Operands, ra, src);

    copy_back_(&dst->int32Operands, ra, src);

    copy_back_(&dst->quant8AsymmOperands, ra, src);

    copy_back_(&dst->quant16SymmOperands, ra, src);

    copy_back_(&dst->float16Operands, ra, src);

    copy_back_(&dst->bool8Operands, ra, src);

    copy_back_(&dst->quant8ChannelOperands, ra, src);

    copy_back_(&dst->quant16AsymmOperands, ra, src);

    copy_back_(&dst->quant8SymmOperands, ra, src);

    static_assert(9 == MixedTyped::kNumTypes,

                  "Number of types in MixedTyped changed, but copy_back function wasn't updated");

using ::android::hidl::memory::V1_0::IMemory;



constexpr uint32_t kInputPoolIndex = 0;

constexpr uint32_t kOutputPoolIndex = 1;



Request createRequest(const TestModel& testModel) {

    // Model inputs.

    hidl_vec<RequestArgument> inputs(testModel.inputIndexes.size());

    size_t inputSize = 0;

    for (uint32_t i = 0; i < testModel.inputIndexes.size(); i++) {

        const auto& op = testModel.operands[testModel.inputIndexes[i]];

        if (op.data.size() == 0) {

            // Omitted input.

            inputs[i] = {.hasNoValue = true};

        } else {

            DataLocation loc = {.poolIndex = kInputPoolIndex,

                                .offset = static_cast<uint32_t>(inputSize),

                                .length = static_cast<uint32_t>(op.data.size())};

            inputSize += op.data.alignedSize();

            inputs[i] = {.hasNoValue = false, .location = loc, .dimensions = {}};

        }

    }



    // Model outputs.

    hidl_vec<RequestArgument> outputs(testModel.outputIndexes.size());

    size_t outputSize = 0;

    for (uint32_t i = 0; i < testModel.outputIndexes.size(); i++) {

        const auto& op = testModel.operands[testModel.outputIndexes[i]];



        // In the case of zero-sized output, we should at least provide a one-byte buffer.

        // This is because zero-sized tensors are only supported internally to the driver, or

        // reported in output shapes. It is illegal for the client to pre-specify a zero-sized

        // tensor as model output. Otherwise, we will have two semantic conflicts:

        // - "Zero dimension" conflicts with "unspecified dimension".

        // - "Omitted operand buffer" conflicts with "zero-sized operand buffer".

        size_t bufferSize = std::max<size_t>(op.data.size(), 1);



        DataLocation loc = {.poolIndex = kOutputPoolIndex,

                            .offset = static_cast<uint32_t>(outputSize),

                            .length = static_cast<uint32_t>(bufferSize)};

        outputSize += op.data.size() == 0 ? TestBuffer::kAlignment : op.data.alignedSize();

        outputs[i] = {.hasNoValue = false, .location = loc, .dimensions = {}};

    }



    // Allocate memory pools.

    hidl_vec<hidl_memory> pools = {nn::allocateSharedMemory(inputSize),

                                   nn::allocateSharedMemory(outputSize)};

    CHECK_NE(pools[kInputPoolIndex].size(), 0u);

    CHECK_NE(pools[kOutputPoolIndex].size(), 0u);

    sp<IMemory> inputMemory = mapMemory(pools[kInputPoolIndex]);

    CHECK(inputMemory.get() != nullptr);

    uint8_t* inputPtr = static_cast<uint8_t*>(static_cast<void*>(inputMemory->getPointer()));

    CHECK(inputPtr != nullptr);



    // Copy input data to the memory pool.

    for (uint32_t i = 0; i < testModel.inputIndexes.size(); i++) {

        const auto& op = testModel.operands[testModel.inputIndexes[i]];

        if (op.data.size() > 0) {

            const uint8_t* begin = op.data.get<uint8_t>();

            const uint8_t* end = begin + op.data.size();

            std::copy(begin, end, inputPtr + inputs[i].location.offset);

        }

    }



    return {.inputs = std::move(inputs), .outputs = std::move(outputs), .pools = std::move(pools)};

}



std::vector<TestBuffer> getOutputBuffers(const Request& request) {

    sp<IMemory> outputMemory = mapMemory(request.pools[kOutputPoolIndex]);

    CHECK(outputMemory.get() != nullptr);

    uint8_t* outputPtr = static_cast<uint8_t*>(static_cast<void*>(outputMemory->getPointer()));

    CHECK(outputPtr != nullptr);



    // Copy out output results.

    std::vector<TestBuffer> outputBuffers;

    for (const auto& output : request.outputs) {

        outputBuffers.emplace_back(output.location.length, outputPtr + output.location.offset);

    }



    return outputBuffers;

}



}  // namespace neuralnetworks