Test harness for generated tests (2c4e1368) · Commits · e / os / android_hardware_interfaces

neuralnetworks/1.0/vts/functional/Android.bp

+5 −0

Original line number	Diff line number	Diff line
		@@ -18,6 +18,7 @@ cc_test {
		name: "VtsHalNeuralnetworksV1_0TargetTest",
		srcs: [
		"Event.cpp",
		"GeneratedTestHarness.cpp",
		"VtsHalNeuralnetworksV1_0TargetTest.cpp",
		],
		defaults: ["VtsHalTargetTestDefaults"],
		@@ -27,4 +28,8 @@ cc_test {
		"android.hidl.memory@1.0",
		"libhidlmemory",
		],
		header_libs: [
		"libneuralnetworks_generated_test_harness_headers",
		"libneuralnetworks_generated_tests",
		],
		}

neuralnetworks/1.0/vts/functional/GeneratedTestHarness.cpp

0 → 100644

+191 −0

Original line number	Diff line number	Diff line
		/*
		* Copyright (C) 2017 The Android Open Source Project
		*
		* Licensed under the Apache License, Version 2.0 (the "License");
		* you may not use this file except in compliance with the License.
		* You may obtain a copy of the License at
		*
		* http://www.apache.org/licenses/LICENSE-2.0
		*
		* Unless required by applicable law or agreed to in writing, software
		* distributed under the License is distributed on an "AS IS" BASIS,
		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
		* See the License for the specific language governing permissions and
		* limitations under the License.
		*/

		#include "Event.h"
		#include "TestHarness.h"
		#include "VtsHalNeuralnetworksV1_0TargetTest.h"

		#include <android-base/logging.h>
		#include <android/hidl/memory/1.0/IMemory.h>
		#include <hidlmemory/mapping.h>

		namespace android {
		namespace hardware {
		namespace neuralnetworks {
		namespace V1_0 {
		namespace vts {
		namespace functional {
		// allocator helper
		hidl_memory allocateSharedMemory(int64_t size, const std::string& type = "ashmem");

		namespace generated_tests {
		using ::android::hardware::neuralnetworks::V1_0::implementation::Event;
		using ::generated_tests::for_all;
		using ::generated_tests::for_each;
		using ::generated_tests::resize_accordingly;
		using ::generated_tests::MixedTyped;
		using ::generated_tests::MixedTypedExampleType;
		using ::generated_tests::Float32Operands;
		using ::generated_tests::Int32Operands;
		using ::generated_tests::Quant8Operands;
		// Top level driver for models and examples generated by test_generator.py
		// Test driver for those generated from ml/nn/runtime/test/spec
		void Execute(const sp<IDevice>& device, std::function<Model(void)> create_model,
		const std::vector<MixedTypedExampleType>& examples) {
		Model model = create_model();
		sp<IPreparedModel> preparedModel;
		sp<Event> preparationEvent = new Event();
		ASSERT_NE(nullptr, preparationEvent.get());
		Return<void> prepareRet = device->prepareModel(
		model, preparationEvent, [&](ErrorStatus status, const sp<IPreparedModel>& prepared) {
		EXPECT_EQ(ErrorStatus::NONE, status);
		preparedModel = prepared;
		});
		ASSERT_TRUE(prepareRet.isOk());
		ASSERT_NE(nullptr, preparedModel.get());
		Event::Status preparationStatus = preparationEvent->wait();
		EXPECT_EQ(Event::Status::SUCCESS, preparationStatus);

		const uint32_t INPUT = 0;
		const uint32_t OUTPUT = 1;

		int example_no = 1;
		for (auto& example : examples) {
		SCOPED_TRACE(example_no++);

		const MixedTyped& inputs = example.first;
		const MixedTyped& golden = example.second;

		std::vector<RequestArgument> inputs_info, outputs_info;
		uint32_t inputSize = 0, outputSize = 0;

		// This function only partially specifies the metadata (vector of RequestArguments).
		// The contents are copied over below.
		for_all(inputs, [&inputs_info, &inputSize](int index, auto, auto s) {
		if (inputs_info.size() <= static_cast<size_t>(index)) inputs_info.resize(index + 1);
		RequestArgument arg = {
		.location = {.poolIndex = INPUT, .offset = 0, .length = static_cast<uint32_t>(s)},
		.dimensions = {},
		};
		inputs_info[index] = arg;
		inputSize += s;
		});
		// Compute offset for inputs 1 and so on
		{
		size_t offset = 0;
		for (auto& i : inputs_info) {
		i.location.offset = offset;
		offset += i.location.length;
		}
		}

		MixedTyped test; // holding test results

		// Go through all outputs, initialize RequestArgument descriptors
		resize_accordingly<float>(golden, test);
		resize_accordingly<int32_t>(golden, test);
		resize_accordingly<uint8_t>(golden, test);
		for_all(golden, [&outputs_info, &outputSize](int index, auto, auto s) {
		if (outputs_info.size() <= static_cast<size_t>(index)) outputs_info.resize(index + 1);
		RequestArgument arg = {
		.location = {.poolIndex = OUTPUT, .offset = 0, .length = static_cast<uint32_t>(s)},
		.dimensions = {},
		};
		outputs_info[index] = arg;
		outputSize += s;
		});
		// Compute offset for outputs 1 and so on
		{
		size_t offset = 0;
		for (auto& i : outputs_info) {
		i.location.offset = offset;
		offset += i.location.length;
		}
		}
		std::vector<hidl_memory> pools = {allocateSharedMemory(inputSize),
		allocateSharedMemory(outputSize)};
		ASSERT_NE(0ull, pools[INPUT].size());
		ASSERT_NE(0ull, pools[OUTPUT].size());

		// load data
		sp<IMemory> inputMemory = mapMemory(pools[INPUT]);
		sp<IMemory> outputMemory = mapMemory(pools[OUTPUT]);
		ASSERT_NE(nullptr, inputMemory.get());
		ASSERT_NE(nullptr, outputMemory.get());
		char* inputPtr = reinterpret_cast<char>(static_cast<void>(inputMemory->getPointer()));
		char* outputPtr = reinterpret_cast<char>(static_cast<void>(outputMemory->getPointer()));
		ASSERT_NE(nullptr, inputPtr);
		ASSERT_NE(nullptr, outputPtr);
		inputMemory->update();
		outputMemory->update();

		// Go through all inputs, copy the values
		for_all(inputs, [&inputs_info, inputPtr](int index, auto p, auto s) {
		char* begin = (char*)p;
		char* end = begin + s;
		// TODO: handle more than one input
		std::copy(begin, end, inputPtr + inputs_info[index].location.offset);
		});

		inputMemory->commit();
		outputMemory->commit();
		// execute request
		sp<Event> executionEvent = new Event();
		ASSERT_NE(nullptr, executionEvent.get());
		Return<ErrorStatus> executeStatus = preparedModel->execute(
		{.inputs = inputs_info, .outputs = outputs_info, .pools = pools}, executionEvent);
		ASSERT_TRUE(executeStatus.isOk());
		EXPECT_EQ(ErrorStatus::NONE, static_cast<ErrorStatus>(executeStatus));
		Event::Status eventStatus = executionEvent->wait();
		EXPECT_EQ(Event::Status::SUCCESS, eventStatus);

		// validate results
		outputMemory->read();
		#define COPY_BACK(ty) \
		for_each<ty>(test, [&outputs_info, outputPtr](int index, std::vector<ty>& m) { \
		RequestArgument& i = outputs_info[index]; \
		ASSERT_EQ(m.size(), i.location.length / sizeof(ty)); \
		char* begin = outputPtr + i.location.offset; \
		memcpy(m.data(), begin, i.location.length); \
		});
		COPY_BACK(float);
		COPY_BACK(int32_t);
		COPY_BACK(uint8_t);
		#undef COPY_BACK
		outputMemory->commit();
		// We want "close-enough" results for float
		for_each<float>(golden, [&test](int index, auto& golden_float) {
		auto& test_float_operands = std::get<Float32Operands>(test);
		auto& test_float = test_float_operands[index];
		for (unsigned int i = 0; i < golden_float.size(); i++) {
		SCOPED_TRACE(i);
		EXPECT_FLOAT_EQ(golden_float[i], test_float[i]);
		}
		});

		EXPECT_EQ(std::get<Int32Operands>(golden), std::get<Int32Operands>(test));
		EXPECT_EQ(std::get<Quant8Operands>(golden), std::get<Quant8Operands>(test));
		}
		}

		} // namespace generated_tests

		} // namespace functional
		} // namespace vts
		} // namespace V1_0
		} // namespace neuralnetworks
		} // namespace hardware
		} // namespace android

neuralnetworks/1.0/vts/functional/VtsHalNeuralnetworksV1_0TargetTest.cpp

+21 −16

Original line number	Diff line number	Diff line
		@@ -16,12 +16,13 @@

		#define LOG_TAG "neuralnetworks_hidl_hal_test"

		#include "Event.h"
		#include "VtsHalNeuralnetworksV1_0TargetTest.h"
		#include "Event.h"
		#include "TestHarness.h"

		#include <android-base/logging.h>
		#include <android/hidl/memory/1.0/IMemory.h>
		#include <hidlmemory/mapping.h>
		#include <string>

		namespace android {
		namespace hardware {
		@@ -31,6 +32,11 @@ namespace vts {
		namespace functional {

		using ::android::hardware::neuralnetworks::V1_0::implementation::Event;
		using ::generated_tests::MixedTypedExampleType;
		namespace generated_tests {
		extern void Execute(const sp<IDevice>&, std::function<Model(void)>,
		const std::vector<MixedTypedExampleType>&);
		}

		// A class for test environment setup
		NeuralnetworksHidlEnvironment::NeuralnetworksHidlEnvironment() {}
		@@ -107,9 +113,7 @@ Model createTestModel() {
		.scale = 0.0f,
		.zeroPoint = 0,
		.lifetime = OperandLifeTime::MODEL_INPUT,
		.location = {.poolIndex = 0,
		.offset = 0,
		.length = 0},
		.location = {.poolIndex = 0, .offset = 0, .length = 0},
		},
		{
		.type = OperandType::TENSOR_FLOAT32,
		@@ -118,9 +122,7 @@ Model createTestModel() {
		.scale = 0.0f,
		.zeroPoint = 0,
		.lifetime = OperandLifeTime::CONSTANT_COPY,
		.location = {.poolIndex = 0,
		.offset = 0,
		.length = size},
		.location = {.poolIndex = 0, .offset = 0, .length = size},
		},
		{
		.type = OperandType::INT32,
		@@ -129,9 +131,7 @@ Model createTestModel() {
		.scale = 0.0f,
		.zeroPoint = 0,
		.lifetime = OperandLifeTime::CONSTANT_COPY,
		.location = {.poolIndex = 0,
		.offset = size,
		.length = sizeof(int32_t)},
		.location = {.poolIndex = 0, .offset = size, .length = sizeof(int32_t)},
		},
		{
		.type = OperandType::TENSOR_FLOAT32,
		@@ -140,9 +140,7 @@ Model createTestModel() {
		.scale = 0.0f,
		.zeroPoint = 0,
		.lifetime = OperandLifeTime::MODEL_OUTPUT,
		.location = {.poolIndex = 0,
		.offset = 0,
		.length = 0},
		.location = {.poolIndex = 0, .offset = 0, .length = 0},
		},
		};

		@@ -172,6 +170,7 @@ Model createTestModel() {
		.pools = pools,
		};
		}
		} // anonymous namespace

		// allocator helper
		hidl_memory allocateSharedMemory(int64_t size, const std::string& type = "ashmem") {
		@@ -192,7 +191,6 @@ hidl_memory allocateSharedMemory(int64_t size, const std::string& type = "ashmem

		return memory;
		}
		} // anonymous namespace

		// supported subgraph test
		TEST_F(NeuralnetworksHidlTest, SupportedOperationsTest) {
		@@ -275,8 +273,15 @@ TEST_F(NeuralnetworksHidlTest, SimpleExecuteGraphTest) {
		EXPECT_EQ(expectedData, outputData);
		}

		// Mixed-typed examples
		typedef MixedTypedExampleType MixedTypedExample;

		// in frameworks/ml/nn/runtime/tests/generated/
		#include "all_generated_vts_tests.cpp"

		// TODO: Add tests for execution failure, or wait_for/wait_until timeout.
		// Discussion: https://googleplex-android-review.git.corp.google.com/#/c/platform/hardware/interfaces/+/2654636/5/neuralnetworks/1.0/vts/functional/VtsHalNeuralnetworksV1_0TargetTest.cpp@222
		// Discussion:
		// https://googleplex-android-review.git.corp.google.com/#/c/platform/hardware/interfaces/+/2654636/5/neuralnetworks/1.0/vts/functional/VtsHalNeuralnetworksV1_0TargetTest.cpp@222

		} // namespace functional
		} // namespace vts