Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit d128dc15 authored by Michael Butler's avatar Michael Butler Committed by android-build-merger
Browse files

Follow up CL to "Add validation tests for NNAPI Burst serialized format" 

am: 353a624b

Change-Id: Ia73cd893c5b76957fcf74343649704fc2f8f2c1c
parents d496c348 353a624b

neuralnetworks/1.0/vts/functional/ValidateRequest.cpp

+1 −56
Original line number Diff line number Diff line
@@ -34,7 +34,6 @@ namespace vts { 
namespace functional {



using ::android::hardware::neuralnetworks::V1_2::implementation::ExecutionCallback;

using ::android::hardware::neuralnetworks::V1_2::implementation::PreparedModelCallback;

using ::android::hidl::memory::V1_0::IMemory;

using test_helper::for_all;

using test_helper::MixedTyped;
@@ -42,53 +41,6 @@ using test_helper::MixedTypedExample; 


///////////////////////// UTILITY FUNCTIONS /////////////////////////



static void createPreparedModel(const sp<IDevice>& device, const V1_0::Model& model,

                                sp<IPreparedModel>* preparedModel) {

    ASSERT_NE(nullptr, preparedModel);



    // see if service can handle model

    bool fullySupportsModel = false;

    Return<void> supportedOpsLaunchStatus = device->getSupportedOperations(

        model, [&fullySupportsModel](ErrorStatus status, const hidl_vec<bool>& supported) {

            ASSERT_EQ(ErrorStatus::NONE, status);

            ASSERT_NE(0ul, supported.size());

            fullySupportsModel =

                std::all_of(supported.begin(), supported.end(), [](bool valid) { return valid; });

        });

    ASSERT_TRUE(supportedOpsLaunchStatus.isOk());



    // launch prepare model

    sp<PreparedModelCallback> preparedModelCallback = new PreparedModelCallback();

    ASSERT_NE(nullptr, preparedModelCallback.get());

    Return<ErrorStatus> prepareLaunchStatus = device->prepareModel(model, preparedModelCallback);

    ASSERT_TRUE(prepareLaunchStatus.isOk());

    ASSERT_EQ(ErrorStatus::NONE, static_cast<ErrorStatus>(prepareLaunchStatus));



    // retrieve prepared model

    preparedModelCallback->wait();

    ErrorStatus prepareReturnStatus = preparedModelCallback->getStatus();

    *preparedModel = preparedModelCallback->getPreparedModel();



    // The getSupportedOperations call returns a list of operations that are

    // guaranteed not to fail if prepareModel is called, and

    // 'fullySupportsModel' is true i.f.f. the entire model is guaranteed.

    // If a driver has any doubt that it can prepare an operation, it must

    // return false. So here, if a driver isn't sure if it can support an

    // operation, but reports that it successfully prepared the model, the test

    // can continue.

    if (!fullySupportsModel && prepareReturnStatus != ErrorStatus::NONE) {

        ASSERT_EQ(nullptr, preparedModel->get());

        LOG(INFO) << "NN VTS: Unable to test Request validation because vendor service cannot "

                     "prepare model that it does not support.";

        std::cout << "[          ]   Unable to test Request validation because vendor service "

                     "cannot prepare model that it does not support."

                  << std::endl;

        return;

    }

    ASSERT_EQ(ErrorStatus::NONE, prepareReturnStatus);

    ASSERT_NE(nullptr, preparedModel->get());

}



// Primary validation function. This function will take a valid request, apply a

// mutation to it to invalidate the request, then pass it to interface calls

// that use the request. Note that the request here is passed by value, and any
@@ -237,15 +189,8 @@ std::vector<Request> createRequests(const std::vector<MixedTypedExample>& exampl 
    return requests;

}



void ValidationTest::validateRequests(const V1_0::Model& model,

void ValidationTest::validateRequests(const sp<IPreparedModel>& preparedModel,

                                      const std::vector<Request>& requests) {

    // create IPreparedModel

    sp<IPreparedModel> preparedModel;

    ASSERT_NO_FATAL_FAILURE(createPreparedModel(device, model, &preparedModel));

    if (preparedModel == nullptr) {

        return;

    }



    // validate each request

    for (const Request& request : requests) {

        removeInputTest(preparedModel, request);

neuralnetworks/1.0/vts/functional/VtsHalNeuralnetworks.cpp

+63 −2
Original line number Diff line number Diff line
@@ -18,6 +18,10 @@ 


#include "VtsHalNeuralnetworks.h"



#include <android-base/logging.h>



#include "Callbacks.h"



namespace android {

namespace hardware {

namespace neuralnetworks {
@@ -25,6 +29,55 @@ namespace V1_0 { 
namespace vts {

namespace functional {



using ::android::hardware::neuralnetworks::V1_2::implementation::PreparedModelCallback;



static void createPreparedModel(const sp<IDevice>& device, const V1_0::Model& model,

                                sp<IPreparedModel>* preparedModel) {

    ASSERT_NE(nullptr, preparedModel);



    // see if service can handle model

    bool fullySupportsModel = false;

    Return<void> supportedOpsLaunchStatus = device->getSupportedOperations(

            model, [&fullySupportsModel](ErrorStatus status, const hidl_vec<bool>& supported) {

                ASSERT_EQ(ErrorStatus::NONE, status);

                ASSERT_NE(0ul, supported.size());

                fullySupportsModel = std::all_of(supported.begin(), supported.end(),

                                                 [](bool valid) { return valid; });

            });

    ASSERT_TRUE(supportedOpsLaunchStatus.isOk());



    // launch prepare model

    sp<PreparedModelCallback> preparedModelCallback = new PreparedModelCallback();

    ASSERT_NE(nullptr, preparedModelCallback.get());

    Return<ErrorStatus> prepareLaunchStatus = device->prepareModel(model, preparedModelCallback);

    ASSERT_TRUE(prepareLaunchStatus.isOk());

    ASSERT_EQ(ErrorStatus::NONE, static_cast<ErrorStatus>(prepareLaunchStatus));



    // retrieve prepared model

    preparedModelCallback->wait();

    ErrorStatus prepareReturnStatus = preparedModelCallback->getStatus();

    *preparedModel = preparedModelCallback->getPreparedModel();



    // The getSupportedOperations call returns a list of operations that are

    // guaranteed not to fail if prepareModel is called, and

    // 'fullySupportsModel' is true i.f.f. the entire model is guaranteed.

    // If a driver has any doubt that it can prepare an operation, it must

    // return false. So here, if a driver isn't sure if it can support an

    // operation, but reports that it successfully prepared the model, the test

    // can continue.

    if (!fullySupportsModel && prepareReturnStatus != ErrorStatus::NONE) {

        ASSERT_EQ(nullptr, preparedModel->get());

        LOG(INFO) << "NN VTS: Unable to test Request validation because vendor service cannot "

                     "prepare model that it does not support.";

        std::cout << "[          ]   Unable to test Request validation because vendor service "

                     "cannot prepare model that it does not support."

                  << std::endl;

        return;

    }

    ASSERT_EQ(ErrorStatus::NONE, prepareReturnStatus);

    ASSERT_NE(nullptr, preparedModel->get());

}



// A class for test environment setup

NeuralnetworksHidlEnvironment::NeuralnetworksHidlEnvironment() {}
@@ -68,9 +121,17 @@ void NeuralnetworksHidlTest::TearDown() { 
    ::testing::VtsHalHidlTargetTestBase::TearDown();

}



void ValidationTest::validateEverything(const Model& model, const std::vector<Request>& request) {

void ValidationTest::validateEverything(const Model& model, const std::vector<Request>& requests) {

    validateModel(model);

    validateRequests(model, request);



    // create IPreparedModel

    sp<IPreparedModel> preparedModel;

    ASSERT_NO_FATAL_FAILURE(createPreparedModel(device, model, &preparedModel));

    if (preparedModel == nullptr) {

        return;

    }



    validateRequests(preparedModel, requests);

}



}  // namespace functional

neuralnetworks/1.0/vts/functional/VtsHalNeuralnetworks.h

+2 −1
Original line number Diff line number Diff line
@@ -67,7 +67,8 @@ class ValidationTest : public NeuralnetworksHidlTest { 


   private:

     void validateModel(const Model& model);

     void validateRequests(const Model& model, const std::vector<Request>& request);

     void validateRequests(const sp<IPreparedModel>& preparedModel,

                           const std::vector<Request>& requests);

};



// Tag for the generated tests

neuralnetworks/1.1/vts/functional/ValidateRequest.cpp

+1 −57
Original line number Diff line number Diff line
@@ -34,7 +34,6 @@ namespace vts { 
namespace functional {



using ::android::hardware::neuralnetworks::V1_2::implementation::ExecutionCallback;

using ::android::hardware::neuralnetworks::V1_2::implementation::PreparedModelCallback;

using ::android::hidl::memory::V1_0::IMemory;

using test_helper::for_all;

using test_helper::MixedTyped;
@@ -42,54 +41,6 @@ using test_helper::MixedTypedExample; 


///////////////////////// UTILITY FUNCTIONS /////////////////////////



static void createPreparedModel(const sp<IDevice>& device, const V1_1::Model& model,

                                sp<IPreparedModel>* preparedModel) {

    ASSERT_NE(nullptr, preparedModel);



    // see if service can handle model

    bool fullySupportsModel = false;

    Return<void> supportedOpsLaunchStatus = device->getSupportedOperations_1_1(

        model, [&fullySupportsModel](ErrorStatus status, const hidl_vec<bool>& supported) {

            ASSERT_EQ(ErrorStatus::NONE, status);

            ASSERT_NE(0ul, supported.size());

            fullySupportsModel =

                std::all_of(supported.begin(), supported.end(), [](bool valid) { return valid; });

        });

    ASSERT_TRUE(supportedOpsLaunchStatus.isOk());



    // launch prepare model

    sp<PreparedModelCallback> preparedModelCallback = new PreparedModelCallback();

    ASSERT_NE(nullptr, preparedModelCallback.get());

    Return<ErrorStatus> prepareLaunchStatus = device->prepareModel_1_1(

        model, ExecutionPreference::FAST_SINGLE_ANSWER, preparedModelCallback);

    ASSERT_TRUE(prepareLaunchStatus.isOk());

    ASSERT_EQ(ErrorStatus::NONE, static_cast<ErrorStatus>(prepareLaunchStatus));



    // retrieve prepared model

    preparedModelCallback->wait();

    ErrorStatus prepareReturnStatus = preparedModelCallback->getStatus();

    *preparedModel = preparedModelCallback->getPreparedModel();



    // The getSupportedOperations_1_1 call returns a list of operations that are

    // guaranteed not to fail if prepareModel_1_1 is called, and

    // 'fullySupportsModel' is true i.f.f. the entire model is guaranteed.

    // If a driver has any doubt that it can prepare an operation, it must

    // return false. So here, if a driver isn't sure if it can support an

    // operation, but reports that it successfully prepared the model, the test

    // can continue.

    if (!fullySupportsModel && prepareReturnStatus != ErrorStatus::NONE) {

        ASSERT_EQ(nullptr, preparedModel->get());

        LOG(INFO) << "NN VTS: Unable to test Request validation because vendor service cannot "

                     "prepare model that it does not support.";

        std::cout << "[          ]   Unable to test Request validation because vendor service "

                     "cannot prepare model that it does not support."

                  << std::endl;

        return;

    }

    ASSERT_EQ(ErrorStatus::NONE, prepareReturnStatus);

    ASSERT_NE(nullptr, preparedModel->get());

}



// Primary validation function. This function will take a valid request, apply a

// mutation to it to invalidate the request, then pass it to interface calls

// that use the request. Note that the request here is passed by value, and any
@@ -238,15 +189,8 @@ std::vector<Request> createRequests(const std::vector<MixedTypedExample>& exampl 
    return requests;

}



void ValidationTest::validateRequests(const V1_1::Model& model,

void ValidationTest::validateRequests(const sp<IPreparedModel>& preparedModel,

                                      const std::vector<Request>& requests) {

    // create IPreparedModel

    sp<IPreparedModel> preparedModel;

    ASSERT_NO_FATAL_FAILURE(createPreparedModel(device, model, &preparedModel));

    if (preparedModel == nullptr) {

        return;

    }



    // validate each request

    for (const Request& request : requests) {

        removeInputTest(preparedModel, request);

neuralnetworks/1.1/vts/functional/VtsHalNeuralnetworks.cpp

+64 −2
Original line number Diff line number Diff line
@@ -18,6 +18,10 @@ 


#include "VtsHalNeuralnetworks.h"



#include <android-base/logging.h>



#include "Callbacks.h"



namespace android {

namespace hardware {

namespace neuralnetworks {
@@ -25,6 +29,56 @@ namespace V1_1 { 
namespace vts {

namespace functional {



using ::android::hardware::neuralnetworks::V1_2::implementation::PreparedModelCallback;



static void createPreparedModel(const sp<IDevice>& device, const V1_1::Model& model,

                                sp<IPreparedModel>* preparedModel) {

    ASSERT_NE(nullptr, preparedModel);



    // see if service can handle model

    bool fullySupportsModel = false;

    Return<void> supportedOpsLaunchStatus = device->getSupportedOperations_1_1(

            model, [&fullySupportsModel](ErrorStatus status, const hidl_vec<bool>& supported) {

                ASSERT_EQ(ErrorStatus::NONE, status);

                ASSERT_NE(0ul, supported.size());

                fullySupportsModel = std::all_of(supported.begin(), supported.end(),

                                                 [](bool valid) { return valid; });

            });

    ASSERT_TRUE(supportedOpsLaunchStatus.isOk());



    // launch prepare model

    sp<PreparedModelCallback> preparedModelCallback = new PreparedModelCallback();

    ASSERT_NE(nullptr, preparedModelCallback.get());

    Return<ErrorStatus> prepareLaunchStatus = device->prepareModel_1_1(

            model, ExecutionPreference::FAST_SINGLE_ANSWER, preparedModelCallback);

    ASSERT_TRUE(prepareLaunchStatus.isOk());

    ASSERT_EQ(ErrorStatus::NONE, static_cast<ErrorStatus>(prepareLaunchStatus));



    // retrieve prepared model

    preparedModelCallback->wait();

    ErrorStatus prepareReturnStatus = preparedModelCallback->getStatus();

    *preparedModel = preparedModelCallback->getPreparedModel();



    // The getSupportedOperations_1_1 call returns a list of operations that are

    // guaranteed not to fail if prepareModel_1_1 is called, and

    // 'fullySupportsModel' is true i.f.f. the entire model is guaranteed.

    // If a driver has any doubt that it can prepare an operation, it must

    // return false. So here, if a driver isn't sure if it can support an

    // operation, but reports that it successfully prepared the model, the test

    // can continue.

    if (!fullySupportsModel && prepareReturnStatus != ErrorStatus::NONE) {

        ASSERT_EQ(nullptr, preparedModel->get());

        LOG(INFO) << "NN VTS: Unable to test Request validation because vendor service cannot "

                     "prepare model that it does not support.";

        std::cout << "[          ]   Unable to test Request validation because vendor service "

                     "cannot prepare model that it does not support."

                  << std::endl;

        return;

    }

    ASSERT_EQ(ErrorStatus::NONE, prepareReturnStatus);

    ASSERT_NE(nullptr, preparedModel->get());

}



// A class for test environment setup

NeuralnetworksHidlEnvironment::NeuralnetworksHidlEnvironment() {}
@@ -68,9 +122,17 @@ void NeuralnetworksHidlTest::TearDown() { 
    ::testing::VtsHalHidlTargetTestBase::TearDown();

}



void ValidationTest::validateEverything(const Model& model, const std::vector<Request>& request) {

void ValidationTest::validateEverything(const Model& model, const std::vector<Request>& requests) {

    validateModel(model);

    validateRequests(model, request);



    // create IPreparedModel

    sp<IPreparedModel> preparedModel;

    ASSERT_NO_FATAL_FAILURE(createPreparedModel(device, model, &preparedModel));

    if (preparedModel == nullptr) {

        return;

    }



    validateRequests(preparedModel, requests);

}



}  // namespace functional