Modify 1.1 VTS tests to consume test struct directly. (37f937b7) · Commits · e / os / android_hardware_interfaces

neuralnetworks/1.1/vts/functional/Android.bp

+1 −2

Original line number	Diff line number	Diff line
		@@ -34,13 +34,12 @@ cc_defaults {
		"android.hidl.memory@1.0",
		"libgmock",
		"libhidlmemory",
		"libneuralnetworks_generated_test_harness",
		"libneuralnetworks_utils",
		"VtsHalNeuralNetworksV1_0_utils",
		],
		header_libs: [
		"libneuralnetworks_headers",
		"libneuralnetworks_generated_test_harness_headers",
		"libneuralnetworks_generated_tests",
		],
		test_suites: ["general-tests"],
		}

neuralnetworks/1.1/vts/functional/GeneratedTestHarness.cpp

+103 −131

Original line number	Diff line number	Diff line
		@@ -24,6 +24,7 @@
		#include <android/hidl/memory/1.0/IMemory.h>
		#include <hidlmemory/mapping.h>

		#include <gtest/gtest.h>
		#include <iostream>

		#include "1.0/Callbacks.h"
		@@ -37,8 +38,13 @@ namespace neuralnetworks {
		namespace V1_1 {
		namespace generated_tests {

		using namespace test_helper;
		using ::android::hardware::neuralnetworks::V1_0::DataLocation;
		using ::android::hardware::neuralnetworks::V1_0::ErrorStatus;
		using ::android::hardware::neuralnetworks::V1_0::IPreparedModel;
		using ::android::hardware::neuralnetworks::V1_0::Operand;
		using ::android::hardware::neuralnetworks::V1_0::OperandLifeTime;
		using ::android::hardware::neuralnetworks::V1_0::OperandType;
		using ::android::hardware::neuralnetworks::V1_0::Request;
		using ::android::hardware::neuralnetworks::V1_0::RequestArgument;
		using ::android::hardware::neuralnetworks::V1_0::implementation::ExecutionCallback;
		@@ -47,144 +53,112 @@ using ::android::hardware::neuralnetworks::V1_1::ExecutionPreference;
		using ::android::hardware::neuralnetworks::V1_1::IDevice;
		using ::android::hardware::neuralnetworks::V1_1::Model;
		using ::android::hidl::memory::V1_0::IMemory;
		using ::test_helper::compare;
		using ::test_helper::filter;
		using ::test_helper::for_all;
		using ::test_helper::MixedTyped;
		using ::test_helper::MixedTypedExample;
		using ::test_helper::resize_accordingly;

		// Top level driver for models and examples generated by test_generator.py
		// Test driver for those generated from ml/nn/runtime/test/spec
		void EvaluatePreparedModel(sp<IPreparedModel>& preparedModel, std::function<bool(int)> is_ignored,
		const std::vector<MixedTypedExample>& examples,
		bool hasRelaxedFloat32Model, float fpAtol, float fpRtol) {
		const uint32_t INPUT = 0;
		const uint32_t OUTPUT = 1;

		int example_no = 1;
		for (auto& example : examples) {
		SCOPED_TRACE(example_no++);
		const MixedTyped& inputs = example.operands.first;
		const MixedTyped& golden = example.operands.second;

		const bool hasFloat16Inputs = !inputs.float16Operands.empty();
		if (hasRelaxedFloat32Model \|\| hasFloat16Inputs) {
		// TODO: Adjust the error limit based on testing.
		// If in relaxed mode, set the absolute tolerance to be 5ULP of FP16.
		fpAtol = 5.0f * 0.0009765625f;
		// Set the relative tolerance to be 5ULP of the corresponding FP precision.
		fpRtol = 5.0f * 0.0009765625f;
		Model createModel(const TestModel& testModel) {
		// Model operands.
		hidl_vec<Operand> operands(testModel.operands.size());
		size_t constCopySize = 0, constRefSize = 0;
		for (uint32_t i = 0; i < testModel.operands.size(); i++) {
		const auto& op = testModel.operands[i];

		DataLocation loc = {};
		if (op.lifetime == TestOperandLifeTime::CONSTANT_COPY) {
		loc = {.poolIndex = 0,
		.offset = static_cast<uint32_t>(constCopySize),
		.length = static_cast<uint32_t>(op.data.size())};
		constCopySize += op.data.alignedSize();
		} else if (op.lifetime == TestOperandLifeTime::CONSTANT_REFERENCE) {
		loc = {.poolIndex = 0,
		.offset = static_cast<uint32_t>(constRefSize),
		.length = static_cast<uint32_t>(op.data.size())};
		constRefSize += op.data.alignedSize();
		}

		std::vector<RequestArgument> inputs_info, outputs_info;
		uint32_t inputSize = 0, outputSize = 0;
		// This function only partially specifies the metadata (vector of RequestArguments).
		// The contents are copied over below.
		for_all(inputs, [&inputs_info, &inputSize](int index, auto, auto s) {
		if (inputs_info.size() <= static_cast<size_t>(index)) inputs_info.resize(index + 1);
		RequestArgument arg = {
		.location = {.poolIndex = INPUT,
		.offset = 0,
		.length = static_cast<uint32_t>(s)},
		.dimensions = {},
		};
		RequestArgument arg_empty = {
		.hasNoValue = true,
		};
		inputs_info[index] = s ? arg : arg_empty;
		inputSize += s;
		});
		// Compute offset for inputs 1 and so on
		{
		size_t offset = 0;
		for (auto& i : inputs_info) {
		if (!i.hasNoValue) i.location.offset = offset;
		offset += i.location.length;
		}
		operands[i] = {.type = static_cast<OperandType>(op.type),
		.dimensions = op.dimensions,
		.numberOfConsumers = op.numberOfConsumers,
		.scale = op.scale,
		.zeroPoint = op.zeroPoint,
		.lifetime = static_cast<OperandLifeTime>(op.lifetime),
		.location = loc};
		}

		MixedTyped test; // holding test results

		// Go through all outputs, initialize RequestArgument descriptors
		resize_accordingly(golden, test);
		for_all(golden, [&outputs_info, &outputSize](int index, auto, auto s) {
		if (outputs_info.size() <= static_cast<size_t>(index)) outputs_info.resize(index + 1);
		RequestArgument arg = {
		.location = {.poolIndex = OUTPUT,
		.offset = 0,
		.length = static_cast<uint32_t>(s)},
		.dimensions = {},
		};
		outputs_info[index] = arg;
		outputSize += s;
		// Model operations.
		hidl_vec<Operation> operations(testModel.operations.size());
		std::transform(testModel.operations.begin(), testModel.operations.end(), operations.begin(),
		[](const TestOperation& op) -> Operation {
		return {.type = static_cast<OperationType>(op.type),
		.inputs = op.inputs,
		.outputs = op.outputs};
		});
		// Compute offset for outputs 1 and so on
		{
		size_t offset = 0;
		for (auto& i : outputs_info) {
		i.location.offset = offset;
		offset += i.location.length;

		// Constant copies.
		hidl_vec<uint8_t> operandValues(constCopySize);
		for (uint32_t i = 0; i < testModel.operands.size(); i++) {
		const auto& op = testModel.operands[i];
		if (op.lifetime == TestOperandLifeTime::CONSTANT_COPY) {
		const uint8_t* begin = op.data.get<uint8_t>();
		const uint8_t* end = begin + op.data.size();
		std::copy(begin, end, operandValues.data() + operands[i].location.offset);
		}
		}
		std::vector<hidl_memory> pools = {nn::allocateSharedMemory(inputSize),
		nn::allocateSharedMemory(outputSize)};
		ASSERT_NE(0ull, pools[INPUT].size());
		ASSERT_NE(0ull, pools[OUTPUT].size());

		// Shared memory.
		hidl_vec<hidl_memory> pools;
		if (constRefSize > 0) {
		hidl_vec_push_back(&pools, nn::allocateSharedMemory(constRefSize));
		CHECK_NE(pools[0].size(), 0u);

		// load data
		sp<IMemory> inputMemory = mapMemory(pools[INPUT]);
		sp<IMemory> outputMemory = mapMemory(pools[OUTPUT]);
		ASSERT_NE(nullptr, inputMemory.get());
		ASSERT_NE(nullptr, outputMemory.get());
		char* inputPtr = reinterpret_cast<char>(static_cast<void>(inputMemory->getPointer()));
		char* outputPtr = reinterpret_cast<char>(static_cast<void>(outputMemory->getPointer()));
		ASSERT_NE(nullptr, inputPtr);
		ASSERT_NE(nullptr, outputPtr);
		inputMemory->update();
		outputMemory->update();

		// Go through all inputs, copy the values
		for_all(inputs, [&inputs_info, inputPtr](int index, auto p, auto s) {
		char* begin = (char*)p;
		char* end = begin + s;
		// TODO: handle more than one input
		std::copy(begin, end, inputPtr + inputs_info[index].location.offset);
		});
		sp<IMemory> mappedMemory = mapMemory(pools[0]);
		CHECK(mappedMemory.get() != nullptr);
		uint8_t* mappedPtr =
		reinterpret_cast<uint8_t>(static_cast<void>(mappedMemory->getPointer()));
		CHECK(mappedPtr != nullptr);

		for (uint32_t i = 0; i < testModel.operands.size(); i++) {
		const auto& op = testModel.operands[i];
		if (op.lifetime == TestOperandLifeTime::CONSTANT_REFERENCE) {
		const uint8_t* begin = op.data.get<uint8_t>();
		const uint8_t* end = begin + op.data.size();
		std::copy(begin, end, mappedPtr + operands[i].location.offset);
		}
		}
		}

		inputMemory->commit();
		outputMemory->commit();
		return {.operands = std::move(operands),
		.operations = std::move(operations),
		.inputIndexes = testModel.inputIndexes,
		.outputIndexes = testModel.outputIndexes,
		.operandValues = std::move(operandValues),
		.pools = std::move(pools),
		.relaxComputationFloat32toFloat16 = testModel.isRelaxed};
		}

		const Request request = {.inputs = inputs_info, .outputs = outputs_info, .pools = pools};
		// Top level driver for models and examples generated by test_generator.py
		// Test driver for those generated from ml/nn/runtime/test/spec
		void EvaluatePreparedModel(const sp<IPreparedModel>& preparedModel, const TestModel& testModel) {
		const Request request = createRequest(testModel);

		// launch execution
		// Launch execution.
		sp<ExecutionCallback> executionCallback = new ExecutionCallback();
		ASSERT_NE(nullptr, executionCallback.get());
		Return<ErrorStatus> executionLaunchStatus =
		preparedModel->execute(request, executionCallback);
		Return<ErrorStatus> executionLaunchStatus = preparedModel->execute(request, executionCallback);
		ASSERT_TRUE(executionLaunchStatus.isOk());
		EXPECT_EQ(ErrorStatus::NONE, static_cast<ErrorStatus>(executionLaunchStatus));

		// retrieve execution status
		// Retrieve execution status.
		executionCallback->wait();
		ASSERT_EQ(ErrorStatus::NONE, executionCallback->getStatus());

		// validate results
		outputMemory->read();
		copy_back(&test, outputs_info, outputPtr);
		outputMemory->commit();
		// Filter out don't cares
		MixedTyped filtered_golden = filter(golden, is_ignored);
		MixedTyped filtered_test = filter(test, is_ignored);
		// Retrieve execution results.
		const std::vector<TestBuffer> outputs = getOutputBuffers(request);

		// We want "close-enough" results for float
		compare(filtered_golden, filtered_test, fpAtol, fpRtol);
		}
		// We want "close-enough" results.
		checkResults(testModel, outputs);
		}

		void Execute(const sp<IDevice>& device, std::function<Model(void)> create_model,
		std::function<bool(int)> is_ignored, const std::vector<MixedTypedExample>& examples) {
		Model model = create_model();
		void Execute(const sp<IDevice>& device, const TestModel& testModel) {
		Model model = createModel(testModel);

		// see if service can handle model
		bool fullySupportsModel = false;
		@@ -199,7 +173,6 @@ void Execute(const sp<IDevice>& device, std::function<Model(void)> create_model,

		// launch prepare model
		sp<PreparedModelCallback> preparedModelCallback = new PreparedModelCallback();
		ASSERT_NE(nullptr, preparedModelCallback.get());
		Return<ErrorStatus> prepareLaunchStatus = device->prepareModel_1_1(
		model, ExecutionPreference::FAST_SINGLE_ANSWER, preparedModelCallback);
		ASSERT_TRUE(prepareLaunchStatus.isOk());
		@@ -223,8 +196,7 @@ void Execute(const sp<IDevice>& device, std::function<Model(void)> create_model,
		EXPECT_EQ(ErrorStatus::NONE, prepareReturnStatus);
		ASSERT_NE(nullptr, preparedModel.get());

		EvaluatePreparedModel(preparedModel, is_ignored, examples,
		model.relaxComputationFloat32toFloat16, 1e-5f, 1e-5f);
		EvaluatePreparedModel(preparedModel, testModel);
		}

		} // namespace generated_tests

neuralnetworks/1.1/vts/functional/GeneratedTestHarness.h

+3 −6

Original line number	Diff line number	Diff line
		@@ -18,9 +18,6 @@
		#define ANDROID_HARDWARE_NEURALNETWORKS_V1_1_GENERATED_TEST_HARNESS_H

		#include <android/hardware/neuralnetworks/1.1/IDevice.h>
		#include <android/hardware/neuralnetworks/1.1/types.h>
		#include <functional>
		#include <vector>
		#include "TestHarness.h"

		namespace android {
		@@ -29,9 +26,9 @@ namespace neuralnetworks {
		namespace V1_1 {
		namespace generated_tests {

		void Execute(const sp<V1_1::IDevice>& device, std::function<V1_1::Model(void)> create_model,
		std::function<bool(int)> is_ignored,
		const std::vector<::test_helper::MixedTypedExample>& examples);
		Model createModel(const ::test_helper::TestModel& testModel);

		void Execute(const sp<V1_1::IDevice>& device, const ::test_helper::TestModel& testModel);

		} // namespace generated_tests
		} // namespace V1_1

neuralnetworks/1.1/vts/functional/GeneratedTests.h

+1 −10

Original line number	Diff line number	Diff line
		@@ -14,20 +14,11 @@
		* limitations under the License.
		*/

		#include <android/hidl/memory/1.0/IMemory.h>
		#include <hidlmemory/mapping.h>

		#include "1.0/Utils.h"
		#include "GeneratedTestHarness.h"
		#include "MemoryUtils.h"
		#include "TestHarness.h"
		#include "VtsHalNeuralnetworks.h"

		namespace android::hardware::neuralnetworks::V1_1::vts::functional {

		std::vector<Request> createRequests(const std::vector<::test_helper::MixedTypedExample>& examples);

		} // namespace android::hardware::neuralnetworks::V1_1::vts::functional

		namespace android::hardware::neuralnetworks::V1_1::generated_tests {

		using namespace android::hardware::neuralnetworks::V1_1::vts::functional;

neuralnetworks/1.1/vts/functional/ValidateRequest.cpp

+4 −108

Original line number	Diff line number	Diff line
		@@ -16,14 +16,8 @@

		#define LOG_TAG "neuralnetworks_hidl_hal_test"

		#include <android-base/logging.h>
		#include <android/hidl/memory/1.0/IMemory.h>
		#include <hidlmemory/mapping.h>

		#include "1.0/Callbacks.h"
		#include "1.0/Utils.h"
		#include "MemoryUtils.h"
		#include "TestHarness.h"
		#include "VtsHalNeuralnetworks.h"

		namespace android {
		@@ -35,13 +29,8 @@ namespace functional {

		using ::android::hardware::neuralnetworks::V1_0::ErrorStatus;
		using ::android::hardware::neuralnetworks::V1_0::Request;
		using ::android::hardware::neuralnetworks::V1_0::RequestArgument;
		using ::android::hardware::neuralnetworks::V1_0::implementation::ExecutionCallback;
		using ::android::hardware::neuralnetworks::V1_1::IPreparedModel;
		using ::android::hidl::memory::V1_0::IMemory;
		using ::test_helper::for_all;
		using ::test_helper::MixedTyped;
		using ::test_helper::MixedTypedExample;

		///////////////////////// UTILITY FUNCTIONS /////////////////////////

		@@ -87,104 +76,11 @@ static void removeOutputTest(const sp<IPreparedModel>& preparedModel, const Requ

		///////////////////////////// ENTRY POINT //////////////////////////////////

		std::vector<Request> createRequests(const std::vector<MixedTypedExample>& examples) {
		const uint32_t INPUT = 0;
		const uint32_t OUTPUT = 1;

		std::vector<Request> requests;

		for (auto& example : examples) {
		const MixedTyped& inputs = example.operands.first;
		const MixedTyped& outputs = example.operands.second;

		std::vector<RequestArgument> inputs_info, outputs_info;
		uint32_t inputSize = 0, outputSize = 0;

		// This function only partially specifies the metadata (vector of RequestArguments).
		// The contents are copied over below.
		for_all(inputs, [&inputs_info, &inputSize](int index, auto, auto s) {
		if (inputs_info.size() <= static_cast<size_t>(index)) inputs_info.resize(index + 1);
		RequestArgument arg = {
		.location = {.poolIndex = INPUT,
		.offset = 0,
		.length = static_cast<uint32_t>(s)},
		.dimensions = {},
		};
		RequestArgument arg_empty = {
		.hasNoValue = true,
		};
		inputs_info[index] = s ? arg : arg_empty;
		inputSize += s;
		});
		// Compute offset for inputs 1 and so on
		{
		size_t offset = 0;
		for (auto& i : inputs_info) {
		if (!i.hasNoValue) i.location.offset = offset;
		offset += i.location.length;
		}
		}

		// Go through all outputs, initialize RequestArgument descriptors
		for_all(outputs, [&outputs_info, &outputSize](int index, auto, auto s) {
		if (outputs_info.size() <= static_cast<size_t>(index)) outputs_info.resize(index + 1);
		RequestArgument arg = {
		.location = {.poolIndex = OUTPUT,
		.offset = 0,
		.length = static_cast<uint32_t>(s)},
		.dimensions = {},
		};
		outputs_info[index] = arg;
		outputSize += s;
		});
		// Compute offset for outputs 1 and so on
		{
		size_t offset = 0;
		for (auto& i : outputs_info) {
		i.location.offset = offset;
		offset += i.location.length;
		}
		}
		std::vector<hidl_memory> pools = {nn::allocateSharedMemory(inputSize),
		nn::allocateSharedMemory(outputSize)};
		if (pools[INPUT].size() == 0 \|\| pools[OUTPUT].size() == 0) {
		return {};
		}

		// map pool
		sp<IMemory> inputMemory = mapMemory(pools[INPUT]);
		if (inputMemory == nullptr) {
		return {};
		}
		char* inputPtr = reinterpret_cast<char>(static_cast<void>(inputMemory->getPointer()));
		if (inputPtr == nullptr) {
		return {};
		}

		// initialize pool
		inputMemory->update();
		for_all(inputs, [&inputs_info, inputPtr](int index, auto p, auto s) {
		char* begin = (char*)p;
		char* end = begin + s;
		// TODO: handle more than one input
		std::copy(begin, end, inputPtr + inputs_info[index].location.offset);
		});
		inputMemory->commit();

		requests.push_back({.inputs = inputs_info, .outputs = outputs_info, .pools = pools});
		}

		return requests;
		}

		void ValidationTest::validateRequests(const sp<IPreparedModel>& preparedModel,
		const std::vector<Request>& requests) {
		// validate each request
		for (const Request& request : requests) {
		void ValidationTest::validateRequest(const sp<IPreparedModel>& preparedModel,
		const Request& request) {
		removeInputTest(preparedModel, request);
		removeOutputTest(preparedModel, request);
		}
		}

		} // namespace functional
		} // namespace vts