Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit e5fe876b authored by Michael Butler's avatar Michael Butler Committed by android-build-merger
Browse files

Relax NeuralNetwork's VTS positive and negative base tests 

am: af5e03a6

Change-Id: Ia9830b7bbd39d692b93265fbeb1f6a395f0b2988
parents d7854aef af5e03a6

neuralnetworks/1.0/vts/functional/GeneratedTestHarness.cpp

+5 −11
Original line number Diff line number Diff line
@@ -72,37 +72,30 @@ void Execute(const sp<IDevice>& device, std::function<Model(void)> create_model, 
    Model model = create_model();



    // see if service can handle model

    ErrorStatus supportedStatus;

    bool fullySupportsModel = false;

    Return<void> supportedCall = device->getSupportedOperations(

        model, [&](ErrorStatus status, const hidl_vec<bool>& supported) {

            supportedStatus = status;

        model, [&fullySupportsModel](ErrorStatus status, const hidl_vec<bool>& supported) {

            ASSERT_EQ(ErrorStatus::NONE, status);

            ASSERT_NE(0ul, supported.size());

            fullySupportsModel =

                std::all_of(supported.begin(), supported.end(), [](bool valid) { return valid; });

        });

    ASSERT_TRUE(supportedCall.isOk());

    ASSERT_EQ(ErrorStatus::NONE, supportedStatus);



    // launch prepare model

    sp<PreparedModelCallback> preparedModelCallback = new PreparedModelCallback();

    ASSERT_NE(nullptr, preparedModelCallback.get());

    Return<ErrorStatus> prepareLaunchStatus = device->prepareModel(model, preparedModelCallback);

    ASSERT_TRUE(prepareLaunchStatus.isOk());

    ASSERT_EQ(ErrorStatus::NONE, static_cast<ErrorStatus>(prepareLaunchStatus));



    // retrieve prepared model

    preparedModelCallback->wait();

    ErrorStatus prepareReturnStatus = preparedModelCallback->getStatus();

    sp<IPreparedModel> preparedModel = preparedModelCallback->getPreparedModel();

    if (fullySupportsModel) {

        EXPECT_EQ(ErrorStatus::NONE, prepareReturnStatus);

    } else {

        EXPECT_TRUE(prepareReturnStatus == ErrorStatus::NONE ||

                    prepareReturnStatus == ErrorStatus::GENERAL_FAILURE);

    }



    // early termination if vendor service cannot fully prepare model

    if (!fullySupportsModel && prepareReturnStatus == ErrorStatus::GENERAL_FAILURE) {

    if (!fullySupportsModel && prepareReturnStatus != ErrorStatus::NONE) {

        ASSERT_EQ(nullptr, preparedModel.get());

        LOG(INFO) << "NN VTS: Early termination of test because vendor service cannot "

                     "prepare model that it does not support.";
@@ -111,6 +104,7 @@ void Execute(const sp<IDevice>& device, std::function<Model(void)> create_model, 
                  << std::endl;

        return;

    }

    EXPECT_EQ(ErrorStatus::NONE, prepareReturnStatus);

    ASSERT_NE(nullptr, preparedModel.get());



    int example_no = 1;

neuralnetworks/1.0/vts/functional/VtsHalNeuralnetworksV1_0TargetTest.cpp

+54 −30
Original line number Diff line number Diff line
@@ -69,26 +69,51 @@ void NeuralnetworksHidlTest::SetUp() { 


void NeuralnetworksHidlTest::TearDown() {}



sp<IPreparedModel> NeuralnetworksHidlTest::doPrepareModelShortcut() {

static void doPrepareModelShortcut(const sp<IDevice>& device, sp<IPreparedModel>* preparedModel) {

    ASSERT_NE(nullptr, preparedModel);

    Model model = createValidTestModel();



    // see if service can handle model

    bool fullySupportsModel = false;

    Return<void> supportedOpsLaunchStatus = device->getSupportedOperations(

        model, [&fullySupportsModel](ErrorStatus status, const hidl_vec<bool>& supported) {

            ASSERT_EQ(ErrorStatus::NONE, status);

            ASSERT_NE(0ul, supported.size());

            fullySupportsModel =

                std::all_of(supported.begin(), supported.end(), [](bool valid) { return valid; });

        });

    ASSERT_TRUE(supportedOpsLaunchStatus.isOk());



    // launch prepare model

    sp<PreparedModelCallback> preparedModelCallback = new PreparedModelCallback();

    if (preparedModelCallback == nullptr) {

        return nullptr;

    }

    ASSERT_NE(nullptr, preparedModelCallback.get());

    Return<ErrorStatus> prepareLaunchStatus = device->prepareModel(model, preparedModelCallback);

    if (!prepareLaunchStatus.isOk() || prepareLaunchStatus != ErrorStatus::NONE) {

        return nullptr;

    }

    ASSERT_TRUE(prepareLaunchStatus.isOk());

    ASSERT_EQ(ErrorStatus::NONE, static_cast<ErrorStatus>(prepareLaunchStatus));



    // retrieve prepared model

    preparedModelCallback->wait();

    ErrorStatus prepareReturnStatus = preparedModelCallback->getStatus();

    sp<IPreparedModel> preparedModel = preparedModelCallback->getPreparedModel();

    if (prepareReturnStatus != ErrorStatus::NONE || preparedModel == nullptr) {

        return nullptr;

    *preparedModel = preparedModelCallback->getPreparedModel();



    // The getSupportedOperations call returns a list of operations that are

    // guaranteed not to fail if prepareModel is called, and

    // 'fullySupportsModel' is true i.f.f. the entire model is guaranteed.

    // If a driver has any doubt that it can prepare an operation, it must

    // return false. So here, if a driver isn't sure if it can support an

    // operation, but reports that it successfully prepared the model, the test

    // can continue.

    if (!fullySupportsModel && prepareReturnStatus != ErrorStatus::NONE) {

        ASSERT_EQ(nullptr, preparedModel->get());

        LOG(INFO) << "NN VTS: Early termination of test because vendor service cannot "

                     "prepare model that it does not support.";

        std::cout << "[          ]   Early termination of test because vendor service cannot "

                     "prepare model that it does not support."

                  << std::endl;

        return;

    }



    return preparedModel;

    ASSERT_EQ(ErrorStatus::NONE, prepareReturnStatus);

    ASSERT_NE(nullptr, preparedModel->get());

}



// create device test
@@ -149,18 +174,8 @@ TEST_F(NeuralnetworksHidlTest, SupportedOperationsNegativeTest2) { 


// prepare simple model positive test

TEST_F(NeuralnetworksHidlTest, SimplePrepareModelPositiveTest) {

    Model model = createValidTestModel();

    sp<PreparedModelCallback> preparedModelCallback = new PreparedModelCallback();

    ASSERT_NE(nullptr, preparedModelCallback.get());

    Return<ErrorStatus> prepareLaunchStatus = device->prepareModel(model, preparedModelCallback);

    ASSERT_TRUE(prepareLaunchStatus.isOk());

    EXPECT_EQ(ErrorStatus::NONE, static_cast<ErrorStatus>(prepareLaunchStatus));



    preparedModelCallback->wait();

    ErrorStatus prepareReturnStatus = preparedModelCallback->getStatus();

    EXPECT_EQ(ErrorStatus::NONE, prepareReturnStatus);

    sp<IPreparedModel> preparedModel = preparedModelCallback->getPreparedModel();

    EXPECT_NE(nullptr, preparedModel.get());

    sp<IPreparedModel> preparedModel;

    doPrepareModelShortcut(device, &preparedModel);

}



// prepare simple model negative test 1
@@ -201,8 +216,11 @@ TEST_F(NeuralnetworksHidlTest, SimpleExecuteGraphPositiveTest) { 
    std::vector<float> expectedData = {6.0f, 8.0f, 10.0f, 12.0f};

    const uint32_t OUTPUT = 1;



    sp<IPreparedModel> preparedModel = doPrepareModelShortcut();

    ASSERT_NE(nullptr, preparedModel.get());

    sp<IPreparedModel> preparedModel;

    ASSERT_NO_FATAL_FAILURE(doPrepareModelShortcut(device, &preparedModel));

    if (preparedModel == nullptr) {

        return;

    }

    Request request = createValidTestRequest();



    auto postWork = [&] {
@@ -235,8 +253,11 @@ TEST_F(NeuralnetworksHidlTest, SimpleExecuteGraphPositiveTest) { 


// execute simple graph negative test 1

TEST_F(NeuralnetworksHidlTest, SimpleExecuteGraphNegativeTest1) {

    sp<IPreparedModel> preparedModel = doPrepareModelShortcut();

    ASSERT_NE(nullptr, preparedModel.get());

    sp<IPreparedModel> preparedModel;

    ASSERT_NO_FATAL_FAILURE(doPrepareModelShortcut(device, &preparedModel));

    if (preparedModel == nullptr) {

        return;

    }

    Request request = createInvalidTestRequest1();



    sp<ExecutionCallback> executionCallback = new ExecutionCallback();
@@ -252,8 +273,11 @@ TEST_F(NeuralnetworksHidlTest, SimpleExecuteGraphNegativeTest1) { 


// execute simple graph negative test 2

TEST_F(NeuralnetworksHidlTest, SimpleExecuteGraphNegativeTest2) {

    sp<IPreparedModel> preparedModel = doPrepareModelShortcut();

    ASSERT_NE(nullptr, preparedModel.get());

    sp<IPreparedModel> preparedModel;

    ASSERT_NO_FATAL_FAILURE(doPrepareModelShortcut(device, &preparedModel));

    if (preparedModel == nullptr) {

        return;

    }

    Request request = createInvalidTestRequest2();



    sp<ExecutionCallback> executionCallback = new ExecutionCallback();

neuralnetworks/1.0/vts/functional/VtsHalNeuralnetworksV1_0TargetTest.h

+0 −2
Original line number Diff line number Diff line
@@ -74,8 +74,6 @@ class NeuralnetworksHidlTest : public ::testing::VtsHalHidlTargetTestBase { 
    void SetUp() override;

    void TearDown() override;



    sp<IPreparedModel> doPrepareModelShortcut();



    sp<IDevice> device;

};