Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 727a7b21 authored by Xusong Wang's avatar Xusong Wang
Browse files

Introduce reusable execution to canonical interface -- HAL. 

This CL modifies the canonical interface for reusable executions:
- Add new interface: IExecution with compute and computeFenced methods
- Add new method IPreparedModel::createExecution

In NNAPI runtime, the new interface IExecution is used to
memoize request-specific execution resources (e.g. converted HAL
request). The expected usage is that, IPreparedModel::createExecution
will be invoked in the first computation of a reusable NDK ANNExecution
object, and IExecution::compute* will be invoked repeatedly.

The IPreparedModel::execute* methods are preserved to avoid redundant
object creation and memoization overhead for a single-time
(non-reusable) execution.

For a vendor implementing the canonical interfaces, only the
IPreparedModel::execute* methods will be called because there is
currently no reusable execution at HAL interface. A DefaultExecution
implementation is provided to reduce the work needed on the vendor side.

Bug: 184073769
Test: NNT_static
Test: neuralnetworks_utils_hal_1_0_test
Test: neuralnetworks_utils_hal_1_1_test
Test: neuralnetworks_utils_hal_1_2_test
Test: neuralnetworks_utils_hal_1_3_test
Test: neuralnetworks_utils_hal_common_test
Test: neuralnetworks_utils_hal_aidl_test
Change-Id: I91790bb5ccf5ae648687fe603f88ffda2c9fd2b2
parent 57c91143

neuralnetworks/1.0/utils/include/nnapi/hal/1.0/Execution.h

0 → 100644
+64 −0
Original line number Diff line number Diff line 
/*

 * Copyright (C) 2021 The Android Open Source Project

 *

 * Licensed under the Apache License, Version 2.0 (the "License");

 * you may not use this file except in compliance with the License.

 * You may obtain a copy of the License at

 *

 *      http://www.apache.org/licenses/LICENSE-2.0

 *

 * Unless required by applicable law or agreed to in writing, software

 * distributed under the License is distributed on an "AS IS" BASIS,

 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

 * See the License for the specific language governing permissions and

 * limitations under the License.

 */



#ifndef ANDROID_HARDWARE_INTERFACES_NEURALNETWORKS_1_0_UTILS_EXECUTION_H

#define ANDROID_HARDWARE_INTERFACES_NEURALNETWORKS_1_0_UTILS_EXECUTION_H



#include <android/hardware/neuralnetworks/1.0/IPreparedModel.h>

#include <nnapi/IExecution.h>

#include <nnapi/Result.h>

#include <nnapi/Types.h>

#include <nnapi/hal/CommonUtils.h>

#include <nnapi/hal/ProtectCallback.h>



#include "PreparedModel.h"



#include <memory>

#include <utility>

#include <vector>



// See hardware/interfaces/neuralnetworks/utils/README.md for more information on HIDL interface

// lifetimes across processes and for protecting asynchronous calls across HIDL.



namespace android::hardware::neuralnetworks::V1_0::utils {



class Execution final : public nn::IExecution, public std::enable_shared_from_this<Execution> {

    struct PrivateConstructorTag {};



  public:

    static nn::GeneralResult<std::shared_ptr<const Execution>> create(

            std::shared_ptr<const PreparedModel> preparedModel, Request request,

            hal::utils::RequestRelocation relocation);



    Execution(PrivateConstructorTag tag, std::shared_ptr<const PreparedModel> preparedModel,

              Request request, hal::utils::RequestRelocation relocation);



    nn::ExecutionResult<std::pair<std::vector<nn::OutputShape>, nn::Timing>> compute(

            const nn::OptionalTimePoint& deadline) const override;



    nn::GeneralResult<std::pair<nn::SyncFence, nn::ExecuteFencedInfoCallback>> computeFenced(

            const std::vector<nn::SyncFence>& waitFor, const nn::OptionalTimePoint& deadline,

            const nn::OptionalDuration& timeoutDurationAfterFence) const override;



  private:

    const std::shared_ptr<const PreparedModel> kPreparedModel;

    const Request kRequest;

    const hal::utils::RequestRelocation kRelocation;

};



}  // namespace android::hardware::neuralnetworks::V1_0::utils



#endif  // ANDROID_HARDWARE_INTERFACES_NEURALNETWORKS_1_0_UTILS_EXECUTION_H

neuralnetworks/1.0/utils/include/nnapi/hal/1.0/PreparedModel.h

+7 −0
Original line number Diff line number Diff line
@@ -57,10 +57,17 @@ class PreparedModel final : public nn::IPreparedModel, 
            const nn::OptionalDuration& loopTimeoutDuration,

            const nn::OptionalDuration& timeoutDurationAfterFence) const override;



    nn::GeneralResult<nn::SharedExecution> createReusableExecution(

            const nn::Request& request, nn::MeasureTiming measure,

            const nn::OptionalDuration& loopTimeoutDuration) const override;



    nn::GeneralResult<nn::SharedBurst> configureExecutionBurst() const override;



    std::any getUnderlyingResource() const override;



    nn::ExecutionResult<std::pair<std::vector<nn::OutputShape>, nn::Timing>> executeInternal(

            const V1_0::Request& request, const hal::utils::RequestRelocation& relocation) const;



  private:

    const sp<V1_0::IPreparedModel> kPreparedModel;

    const hal::utils::DeathHandler kDeathHandler;

neuralnetworks/1.0/utils/src/Execution.cpp

0 → 100644
+72 −0
Original line number Diff line number Diff line 
/*

 * Copyright (C) 2021 The Android Open Source Project

 *

 * Licensed under the Apache License, Version 2.0 (the "License");

 * you may not use this file except in compliance with the License.

 * You may obtain a copy of the License at

 *

 *      http://www.apache.org/licenses/LICENSE-2.0

 *

 * Unless required by applicable law or agreed to in writing, software

 * distributed under the License is distributed on an "AS IS" BASIS,

 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

 * See the License for the specific language governing permissions and

 * limitations under the License.

 */



#include "Execution.h"



#include "Callbacks.h"

#include "Conversions.h"

#include "Utils.h"



#include <android/hardware/neuralnetworks/1.0/IPreparedModel.h>

#include <android/hardware/neuralnetworks/1.0/types.h>

#include <nnapi/IExecution.h>

#include <nnapi/Result.h>

#include <nnapi/TypeUtils.h>

#include <nnapi/Types.h>

#include <nnapi/hal/CommonUtils.h>

#include <nnapi/hal/HandleError.h>

#include <nnapi/hal/ProtectCallback.h>



#include <memory>

#include <utility>

#include <vector>



// See hardware/interfaces/neuralnetworks/utils/README.md for more information on HIDL interface

// lifetimes across processes and for protecting asynchronous calls across HIDL.



namespace android::hardware::neuralnetworks::V1_0::utils {



nn::GeneralResult<std::shared_ptr<const Execution>> Execution::create(

        std::shared_ptr<const PreparedModel> preparedModel, Request request,

        hal::utils::RequestRelocation relocation) {

    if (preparedModel == nullptr) {

        return NN_ERROR() << "V1_0::utils::Execution::create must have non-null preparedModel";

    }



    return std::make_shared<const Execution>(PrivateConstructorTag{}, std::move(preparedModel),

                                             std::move(request), std::move(relocation));

}



Execution::Execution(PrivateConstructorTag /*tag*/,

                     std::shared_ptr<const PreparedModel> preparedModel, Request request,

                     hal::utils::RequestRelocation relocation)

    : kPreparedModel(std::move(preparedModel)),

      kRequest(std::move(request)),

      kRelocation(std::move(relocation)) {}



nn::ExecutionResult<std::pair<std::vector<nn::OutputShape>, nn::Timing>> Execution::compute(

        const nn::OptionalTimePoint& /*deadline*/) const {

    return kPreparedModel->executeInternal(kRequest, kRelocation);

}



nn::GeneralResult<std::pair<nn::SyncFence, nn::ExecuteFencedInfoCallback>> Execution::computeFenced(

        const std::vector<nn::SyncFence>& /*waitFor*/, const nn::OptionalTimePoint& /*deadline*/,

        const nn::OptionalDuration& /*timeoutDurationAfterFence*/) const {

    return NN_ERROR(nn::ErrorStatus::GENERAL_FAILURE)

           << "IExecution::computeFenced is not supported on 1.0 HAL service";

}



}  // namespace android::hardware::neuralnetworks::V1_0::utils

neuralnetworks/1.0/utils/src/PreparedModel.cpp

+32 −6
Original line number Diff line number Diff line
@@ -19,6 +19,7 @@ 
#include "Burst.h"

#include "Callbacks.h"

#include "Conversions.h"

#include "Execution.h"

#include "Utils.h"



#include <android/hardware/neuralnetworks/1.0/IPreparedModel.h>
@@ -61,22 +62,34 @@ nn::ExecutionResult<std::pair<std::vector<nn::OutputShape>, nn::Timing>> Prepare 
        const nn::OptionalDuration& /*loopTimeoutDuration*/) const {

    // Ensure that request is ready for IPC.

    std::optional<nn::Request> maybeRequestInShared;

    const nn::Request& requestInShared = NN_TRY(hal::utils::makeExecutionFailure(

            hal::utils::flushDataFromPointerToShared(&request, &maybeRequestInShared)));

    hal::utils::RequestRelocation relocation;

    const nn::Request& requestInShared =

            NN_TRY(hal::utils::makeExecutionFailure(hal::utils::convertRequestFromPointerToShared(

                    &request, &maybeRequestInShared, &relocation)));



    const auto hidlRequest = NN_TRY(hal::utils::makeExecutionFailure(convert(requestInShared)));



    return executeInternal(hidlRequest, relocation);

}



nn::ExecutionResult<std::pair<std::vector<nn::OutputShape>, nn::Timing>>

PreparedModel::executeInternal(const V1_0::Request& request,

                               const hal::utils::RequestRelocation& relocation) const {

    if (relocation.input) {

        relocation.input->flush();

    }



    const auto cb = sp<ExecutionCallback>::make();

    const auto scoped = kDeathHandler.protectCallback(cb.get());



    const auto ret = kPreparedModel->execute(hidlRequest, cb);

    const auto ret = kPreparedModel->execute(request, cb);

    const auto status = HANDLE_TRANSPORT_FAILURE(ret);

    HANDLE_HAL_STATUS(status) << "execution failed with " << toString(status);



    auto result = NN_TRY(cb->get());

    NN_TRY(hal::utils::makeExecutionFailure(

            hal::utils::unflushDataFromSharedToPointer(request, maybeRequestInShared)));



    if (relocation.output) {

        relocation.output->flush();

    }

    return result;

}
@@ -91,6 +104,19 @@ PreparedModel::executeFenced(const nn::Request& /*request*/, 
           << "IPreparedModel::executeFenced is not supported on 1.0 HAL service";

}



nn::GeneralResult<nn::SharedExecution> PreparedModel::createReusableExecution(

        const nn::Request& request, nn::MeasureTiming /*measure*/,

        const nn::OptionalDuration& /*loopTimeoutDuration*/) const {

    // Ensure that request is ready for IPC.

    std::optional<nn::Request> maybeRequestInShared;

    hal::utils::RequestRelocation relocation;

    const nn::Request& requestInShared = NN_TRY(hal::utils::convertRequestFromPointerToShared(

            &request, &maybeRequestInShared, &relocation));



    auto hidlRequest = NN_TRY(convert(requestInShared));

    return Execution::create(shared_from_this(), std::move(hidlRequest), std::move(relocation));

}



nn::GeneralResult<nn::SharedBurst> PreparedModel::configureExecutionBurst() const {

    return Burst::create(shared_from_this());

}

neuralnetworks/1.0/utils/test/PreparedModelTest.cpp

+145 −0
Original line number Diff line number Diff line
@@ -19,6 +19,7 @@ 
#include <android/hardware/neuralnetworks/1.0/IExecutionCallback.h>

#include <gmock/gmock.h>

#include <gtest/gtest.h>

#include <nnapi/IExecution.h>

#include <nnapi/IPreparedModel.h>

#include <nnapi/TypeUtils.h>

#include <nnapi/Types.h>
@@ -224,6 +225,150 @@ TEST(PreparedModelTest, executeFencedNotSupported) { 
    EXPECT_EQ(result.error().code, nn::ErrorStatus::GENERAL_FAILURE);

}



TEST(PreparedModelTest, reusableExecute) {

    // setup call

    const uint32_t kNumberOfComputations = 2;

    const auto mockPreparedModel = createMockPreparedModel();

    const auto preparedModel = PreparedModel::create(mockPreparedModel).value();

    EXPECT_CALL(*mockPreparedModel, execute(_, _))

            .Times(kNumberOfComputations)

            .WillRepeatedly(Invoke(makeExecute(V1_0::ErrorStatus::NONE, V1_0::ErrorStatus::NONE)));



    // create execution

    const auto createResult = preparedModel->createReusableExecution({}, {}, {});

    ASSERT_TRUE(createResult.has_value())

            << "Failed with " << createResult.error().code << ": " << createResult.error().message;

    ASSERT_NE(createResult.value(), nullptr);



    // invoke compute repeatedly

    for (uint32_t i = 0; i < kNumberOfComputations; i++) {

        const auto computeResult = createResult.value()->compute({});

        EXPECT_TRUE(computeResult.has_value()) << "Failed with " << computeResult.error().code

                                               << ": " << computeResult.error().message;

    }

}



TEST(PreparedModelTest, reusableExecuteLaunchError) {

    // setup test

    const auto mockPreparedModel = createMockPreparedModel();

    const auto preparedModel = PreparedModel::create(mockPreparedModel).value();

    EXPECT_CALL(*mockPreparedModel, execute(_, _))

            .Times(1)

            .WillOnce(Invoke(makeExecute(V1_0::ErrorStatus::GENERAL_FAILURE,

                                         V1_0::ErrorStatus::GENERAL_FAILURE)));



    // create execution

    const auto createResult = preparedModel->createReusableExecution({}, {}, {});

    ASSERT_TRUE(createResult.has_value())

            << "Failed with " << createResult.error().code << ": " << createResult.error().message;

    ASSERT_NE(createResult.value(), nullptr);



    // invoke compute

    const auto computeResult = createResult.value()->compute({});

    ASSERT_FALSE(computeResult.has_value());

    EXPECT_EQ(computeResult.error().code, nn::ErrorStatus::GENERAL_FAILURE);

}



TEST(PreparedModelTest, reusableExecuteReturnError) {

    // setup test

    const auto mockPreparedModel = createMockPreparedModel();

    const auto preparedModel = PreparedModel::create(mockPreparedModel).value();

    EXPECT_CALL(*mockPreparedModel, execute(_, _))

            .Times(1)

            .WillOnce(Invoke(

                    makeExecute(V1_0::ErrorStatus::NONE, V1_0::ErrorStatus::GENERAL_FAILURE)));



    // create execution

    const auto createResult = preparedModel->createReusableExecution({}, {}, {});

    ASSERT_TRUE(createResult.has_value())

            << "Failed with " << createResult.error().code << ": " << createResult.error().message;

    ASSERT_NE(createResult.value(), nullptr);



    // invoke compute

    const auto computeResult = createResult.value()->compute({});

    ASSERT_FALSE(computeResult.has_value());

    EXPECT_EQ(computeResult.error().code, nn::ErrorStatus::GENERAL_FAILURE);

}



TEST(PreparedModelTest, reusableExecuteTransportFailure) {

    // setup test

    const auto mockPreparedModel = createMockPreparedModel();

    const auto preparedModel = PreparedModel::create(mockPreparedModel).value();

    EXPECT_CALL(*mockPreparedModel, execute(_, _))

            .Times(1)

            .WillOnce(InvokeWithoutArgs(makeGeneralTransportFailure));



    // create execution

    const auto createResult = preparedModel->createReusableExecution({}, {}, {});

    ASSERT_TRUE(createResult.has_value())

            << "Failed with " << createResult.error().code << ": " << createResult.error().message;

    ASSERT_NE(createResult.value(), nullptr);



    // invoke compute

    const auto computeResult = createResult.value()->compute({});

    ASSERT_FALSE(computeResult.has_value());

    EXPECT_EQ(computeResult.error().code, nn::ErrorStatus::GENERAL_FAILURE);

}



TEST(PreparedModelTest, reusableExecuteDeadObject) {

    // setup test

    const auto mockPreparedModel = createMockPreparedModel();

    const auto preparedModel = PreparedModel::create(mockPreparedModel).value();

    EXPECT_CALL(*mockPreparedModel, execute(_, _))

            .Times(1)

            .WillOnce(InvokeWithoutArgs(makeDeadObjectFailure));



    // create execution

    const auto createResult = preparedModel->createReusableExecution({}, {}, {});

    ASSERT_TRUE(createResult.has_value())

            << "Failed with " << createResult.error().code << ": " << createResult.error().message;

    ASSERT_NE(createResult.value(), nullptr);



    // invoke compute

    const auto computeResult = createResult.value()->compute({});

    ASSERT_FALSE(computeResult.has_value());

    EXPECT_EQ(computeResult.error().code, nn::ErrorStatus::DEAD_OBJECT);

}



TEST(PreparedModelTest, reusableExecuteCrash) {

    // setup test

    const auto mockPreparedModel = createMockPreparedModel();

    const auto preparedModel = PreparedModel::create(mockPreparedModel).value();

    const auto ret = [&mockPreparedModel]() -> hardware::Return<V1_0::ErrorStatus> {

        mockPreparedModel->simulateCrash();

        return V1_0::ErrorStatus::NONE;

    };

    EXPECT_CALL(*mockPreparedModel, execute(_, _)).Times(1).WillOnce(InvokeWithoutArgs(ret));



    // create execution

    const auto createResult = preparedModel->createReusableExecution({}, {}, {});

    ASSERT_TRUE(createResult.has_value())

            << "Failed with " << createResult.error().code << ": " << createResult.error().message;

    ASSERT_NE(createResult.value(), nullptr);



    // invoke compute

    const auto computeResult = createResult.value()->compute({});

    ASSERT_FALSE(computeResult.has_value());

    EXPECT_EQ(computeResult.error().code, nn::ErrorStatus::DEAD_OBJECT);

}



TEST(PreparedModelTest, reusableExecuteFencedNotSupported) {

    // setup test

    const auto mockPreparedModel = createMockPreparedModel();

    const auto preparedModel = PreparedModel::create(mockPreparedModel).value();



    // create execution

    const auto createResult = preparedModel->createReusableExecution({}, {}, {});

    ASSERT_TRUE(createResult.has_value())

            << "Failed with " << createResult.error().code << ": " << createResult.error().message;

    ASSERT_NE(createResult.value(), nullptr);



    // invoke compute

    const auto computeResult = createResult.value()->computeFenced({}, {}, {});

    ASSERT_FALSE(computeResult.has_value());

    EXPECT_EQ(computeResult.error().code, nn::ErrorStatus::GENERAL_FAILURE);

}



TEST(PreparedModelTest, configureExecutionBurst) {

    // setup test

    const auto mockPreparedModel = MockPreparedModel::create();