Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 35e7599e authored by Xusong Wang's avatar Xusong Wang
Browse files

Add quant8 variant of VTS CompilationCachingTest. 

Prior to this CL, the VTS compilation caching tests are using float32
models. However, there exists several drivers that only support QUANT8
operations, and most of the tests are simply skipped.

This CL makes the CompilationCachingTest fixture parameterized that
every test will run twice, with the first pass running with float32
models and the second pass running with quant8 models.

Bug: 132112463
Test: 1.2 VTS with sample drivers
Test: 1.2 VTS with test drivers that can read and write cache entries
Change-Id: Iaf28065e46c2d90ab8b347b9848be1e6546cce86
parent 85b557fd

neuralnetworks/1.2/vts/functional/CompilationCachingTests.cpp

+148 −43
Original line number Diff line number Diff line
@@ -45,9 +45,9 @@ using ::android::hardware::neuralnetworks::V1_2::implementation::PreparedModelCa 
using ::android::nn::allocateSharedMemory;

using ::test_helper::MixedTypedExample;



namespace {

namespace float32_model {



// In frameworks/ml/nn/runtime/test/generated/, creates a hidl model of mobilenet.

// In frameworks/ml/nn/runtime/test/generated/, creates a hidl model of float32 mobilenet.

#include "examples/mobilenet_224_gender_basic_fixed.example.cpp"

#include "vts_models/mobilenet_224_gender_basic_fixed.model.cpp"
@@ -55,6 +55,44 @@ namespace { 
[[maybe_unused]] auto dummy_createTestModel = createTestModel_dynamic_output_shape;

[[maybe_unused]] auto dummy_get_examples = get_examples_dynamic_output_shape;



// MixedTypedExample is defined in frameworks/ml/nn/tools/test_generator/include/TestHarness.h.

// This function assumes the operation is always ADD.

std::vector<MixedTypedExample> getLargeModelExamples(uint32_t len) {

    float outputValue = 1.0f + static_cast<float>(len);

    return {{.operands = {

                     // Input

                     {.operandDimensions = {{0, {1}}}, .float32Operands = {{0, {1.0f}}}},

                     // Output

                     {.operandDimensions = {{0, {1}}}, .float32Operands = {{0, {outputValue}}}}}}};

}



}  // namespace float32_model



namespace quant8_model {



// In frameworks/ml/nn/runtime/test/generated/, creates a hidl model of quant8 mobilenet.

#include "examples/mobilenet_quantized.example.cpp"

#include "vts_models/mobilenet_quantized.model.cpp"



// Prevent the compiler from complaining about an otherwise unused function.

[[maybe_unused]] auto dummy_createTestModel = createTestModel_dynamic_output_shape;

[[maybe_unused]] auto dummy_get_examples = get_examples_dynamic_output_shape;



// MixedTypedExample is defined in frameworks/ml/nn/tools/test_generator/include/TestHarness.h.

// This function assumes the operation is always ADD.

std::vector<MixedTypedExample> getLargeModelExamples(uint32_t len) {

    uint8_t outputValue = 1 + static_cast<uint8_t>(len);

    return {{.operands = {// Input

                          {.operandDimensions = {{0, {1}}}, .quant8AsymmOperands = {{0, {1}}}},

                          // Output

                          {.operandDimensions = {{0, {1}}},

                           .quant8AsymmOperands = {{0, {outputValue}}}}}}};

}



}  // namespace quant8_model



namespace {



enum class AccessMode { READ_WRITE, READ_ONLY, WRITE_ONLY };



// Creates cache handles based on provided file groups.
@@ -101,14 +139,18 @@ void createCacheHandles(const std::vector<std::vector<std::string>>& fileGroups, 
//                ↑      ↑      ↑             ↑

//               [1]    [1]    [1]           [1]

//

Model createLargeTestModel(OperationType op, uint32_t len) {

// This function assumes the operation is either ADD or MUL.

template <typename CppType, OperandType operandType>

Model createLargeTestModelImpl(OperationType op, uint32_t len) {

    EXPECT_TRUE(op == OperationType::ADD || op == OperationType::MUL);



    // Model operations and operands.

    std::vector<Operation> operations(len);

    std::vector<Operand> operands(len * 2 + 2);



    // The constant buffer pool. This contains the activation scalar, followed by the

    // per-operation constant operands.

    std::vector<uint8_t> operandValues(sizeof(int32_t) + len * sizeof(float));

    std::vector<uint8_t> operandValues(sizeof(int32_t) + len * sizeof(CppType));



    // The activation scalar, value = 0.

    operands[0] = {
@@ -122,7 +164,26 @@ Model createLargeTestModel(OperationType op, uint32_t len) { 
    };

    memset(operandValues.data(), 0, sizeof(int32_t));



    const float floatBufferValue = 1.0f;

    // The buffer value of the constant second operand. The logical value is always 1.0f.

    CppType bufferValue;

    // The scale of the first and second operand.

    float scale1, scale2;

    if (operandType == OperandType::TENSOR_FLOAT32) {

        bufferValue = 1.0f;

        scale1 = 0.0f;

        scale2 = 0.0f;

    } else if (op == OperationType::ADD) {

        bufferValue = 1;

        scale1 = 1.0f;

        scale2 = 1.0f;

    } else {

        // To satisfy the constraint on quant8 MUL: input0.scale * input1.scale < output.scale,

        // set input1 to have scale = 0.5f and bufferValue = 2, i.e. 1.0f in floating point.

        bufferValue = 2;

        scale1 = 1.0f;

        scale2 = 0.5f;

    }



    for (uint32_t i = 0; i < len; i++) {

        const uint32_t firstInputIndex = i * 2 + 1;

        const uint32_t secondInputIndex = firstInputIndex + 1;
@@ -130,10 +191,10 @@ Model createLargeTestModel(OperationType op, uint32_t len) { 


        // The first operation input.

        operands[firstInputIndex] = {

                .type = OperandType::TENSOR_FLOAT32,

                .type = operandType,

                .dimensions = {1},

                .numberOfConsumers = 1,

                .scale = 0.0f,

                .scale = scale1,

                .zeroPoint = 0,

                .lifetime = (i == 0 ? OperandLifeTime::MODEL_INPUT

                                    : OperandLifeTime::TEMPORARY_VARIABLE),
@@ -142,18 +203,18 @@ Model createLargeTestModel(OperationType op, uint32_t len) { 


        // The second operation input, value = 1.

        operands[secondInputIndex] = {

                .type = OperandType::TENSOR_FLOAT32,

                .type = operandType,

                .dimensions = {1},

                .numberOfConsumers = 1,

                .scale = 0.0f,

                .scale = scale2,

                .zeroPoint = 0,

                .lifetime = OperandLifeTime::CONSTANT_COPY,

                .location = {.poolIndex = 0,

                             .offset = static_cast<uint32_t>(i * sizeof(float) + sizeof(int32_t)),

                             .length = sizeof(float)},

                             .offset = static_cast<uint32_t>(i * sizeof(CppType) + sizeof(int32_t)),

                             .length = sizeof(CppType)},

        };

        memcpy(operandValues.data() + sizeof(int32_t) + i * sizeof(float), &floatBufferValue,

               sizeof(float));

        memcpy(operandValues.data() + sizeof(int32_t) + i * sizeof(CppType), &bufferValue,

               sizeof(CppType));



        // The operation. All operations share the same activation scalar.

        // The output operand is created as an input in the next iteration of the loop, in the case
@@ -168,10 +229,10 @@ Model createLargeTestModel(OperationType op, uint32_t len) { 


    // The model output.

    operands.back() = {

            .type = OperandType::TENSOR_FLOAT32,

            .type = operandType,

            .dimensions = {1},

            .numberOfConsumers = 0,

            .scale = 0.0f,

            .scale = scale1,

            .zeroPoint = 0,

            .lifetime = OperandLifeTime::MODEL_OUTPUT,

            .location = {},
@@ -191,22 +252,13 @@ Model createLargeTestModel(OperationType op, uint32_t len) { 
    };

}



// MixedTypedExample is defined in frameworks/ml/nn/tools/test_generator/include/TestHarness.h.

// This function assumes the operation is always ADD.

std::vector<MixedTypedExample> getLargeModelExamples(uint32_t len) {

    float outputValue = 1.0f + static_cast<float>(len);

    return {{.operands = {

                     // Input

                     {.operandDimensions = {{0, {1}}}, .float32Operands = {{0, {1.0f}}}},

                     // Output

                     {.operandDimensions = {{0, {1}}}, .float32Operands = {{0, {outputValue}}}}}}};

};



}  // namespace



// Tag for the compilation caching tests.

class CompilationCachingTest : public NeuralnetworksHidlTest {

class CompilationCachingTestBase : public NeuralnetworksHidlTest {

  protected:

    CompilationCachingTestBase(OperandType type) : kOperandType(type) {}



    void SetUp() override {

        NeuralnetworksHidlTest::SetUp();

        ASSERT_NE(device.get(), nullptr);
@@ -263,6 +315,40 @@ class CompilationCachingTest : public NeuralnetworksHidlTest { 
        NeuralnetworksHidlTest::TearDown();

    }



    // Model and examples creators. According to kOperandType, the following methods will return

    // either float32 model/examples or the quant8 variant.

    Model createTestModel() {

        if (kOperandType == OperandType::TENSOR_FLOAT32) {

            return float32_model::createTestModel();

        } else {

            return quant8_model::createTestModel();

        }

    }



    std::vector<MixedTypedExample> get_examples() {

        if (kOperandType == OperandType::TENSOR_FLOAT32) {

            return float32_model::get_examples();

        } else {

            return quant8_model::get_examples();

        }

    }



    Model createLargeTestModel(OperationType op, uint32_t len) {

        if (kOperandType == OperandType::TENSOR_FLOAT32) {

            return createLargeTestModelImpl<float, OperandType::TENSOR_FLOAT32>(op, len);

        } else {

            return createLargeTestModelImpl<uint8_t, OperandType::TENSOR_QUANT8_ASYMM>(op, len);

        }

    }



    std::vector<MixedTypedExample> getLargeModelExamples(uint32_t len) {

        if (kOperandType == OperandType::TENSOR_FLOAT32) {

            return float32_model::getLargeModelExamples(len);

        } else {

            return quant8_model::getLargeModelExamples(len);

        }

    }



    // See if the service can handle the model.

    bool isModelFullySupported(const V1_2::Model& model) {

        bool fullySupportsModel = false;
@@ -366,9 +452,20 @@ class CompilationCachingTest : public NeuralnetworksHidlTest { 
    uint32_t mNumModelCache;

    uint32_t mNumDataCache;

    uint32_t mIsCachingSupported;



    // The primary data type of the testModel.

    const OperandType kOperandType;

};



TEST_F(CompilationCachingTest, CacheSavingAndRetrieval) {

// A parameterized fixture of CompilationCachingTestBase. Every test will run twice, with the first

// pass running with float32 models and the second pass running with quant8 models.

class CompilationCachingTest : public CompilationCachingTestBase,

                               public ::testing::WithParamInterface<OperandType> {

  protected:

    CompilationCachingTest() : CompilationCachingTestBase(GetParam()) {}

};



TEST_P(CompilationCachingTest, CacheSavingAndRetrieval) {

    // Create test HIDL model and compile.

    const Model testModel = createTestModel();

    if (checkEarlyTermination(testModel)) return;
@@ -409,7 +506,7 @@ TEST_F(CompilationCachingTest, CacheSavingAndRetrieval) { 
                                           /*testDynamicOutputShape=*/false);

}



TEST_F(CompilationCachingTest, CacheSavingAndRetrievalNonZeroOffset) {

TEST_P(CompilationCachingTest, CacheSavingAndRetrievalNonZeroOffset) {

    // Create test HIDL model and compile.

    const Model testModel = createTestModel();

    if (checkEarlyTermination(testModel)) return;
@@ -472,7 +569,7 @@ TEST_F(CompilationCachingTest, CacheSavingAndRetrievalNonZeroOffset) { 
                                           /*testDynamicOutputShape=*/false);

}



TEST_F(CompilationCachingTest, SaveToCacheInvalidNumCache) {

TEST_P(CompilationCachingTest, SaveToCacheInvalidNumCache) {

    // Create test HIDL model and compile.

    const Model testModel = createTestModel();

    if (checkEarlyTermination(testModel)) return;
@@ -584,7 +681,7 @@ TEST_F(CompilationCachingTest, SaveToCacheInvalidNumCache) { 
    }

}



TEST_F(CompilationCachingTest, PrepareModelFromCacheInvalidNumCache) {

TEST_P(CompilationCachingTest, PrepareModelFromCacheInvalidNumCache) {

    // Create test HIDL model and compile.

    const Model testModel = createTestModel();

    if (checkEarlyTermination(testModel)) return;
@@ -664,7 +761,7 @@ TEST_F(CompilationCachingTest, PrepareModelFromCacheInvalidNumCache) { 
    }

}



TEST_F(CompilationCachingTest, SaveToCacheInvalidNumFd) {

TEST_P(CompilationCachingTest, SaveToCacheInvalidNumFd) {

    // Create test HIDL model and compile.

    const Model testModel = createTestModel();

    if (checkEarlyTermination(testModel)) return;
@@ -776,7 +873,7 @@ TEST_F(CompilationCachingTest, SaveToCacheInvalidNumFd) { 
    }

}



TEST_F(CompilationCachingTest, PrepareModelFromCacheInvalidNumFd) {

TEST_P(CompilationCachingTest, PrepareModelFromCacheInvalidNumFd) {

    // Create test HIDL model and compile.

    const Model testModel = createTestModel();

    if (checkEarlyTermination(testModel)) return;
@@ -856,7 +953,7 @@ TEST_F(CompilationCachingTest, PrepareModelFromCacheInvalidNumFd) { 
    }

}



TEST_F(CompilationCachingTest, SaveToCacheInvalidAccessMode) {

TEST_P(CompilationCachingTest, SaveToCacheInvalidAccessMode) {

    // Create test HIDL model and compile.

    const Model testModel = createTestModel();

    if (checkEarlyTermination(testModel)) return;
@@ -914,7 +1011,7 @@ TEST_F(CompilationCachingTest, SaveToCacheInvalidAccessMode) { 
    }

}



TEST_F(CompilationCachingTest, PrepareModelFromCacheInvalidAccessMode) {

TEST_P(CompilationCachingTest, PrepareModelFromCacheInvalidAccessMode) {

    // Create test HIDL model and compile.

    const Model testModel = createTestModel();

    if (checkEarlyTermination(testModel)) return;
@@ -990,7 +1087,7 @@ static void copyCacheFiles(const std::vector<std::vector<std::string>>& from, 
constexpr uint32_t kLargeModelSize = 100;

constexpr uint32_t kNumIterationsTOCTOU = 100;



TEST_F(CompilationCachingTest, SaveToCache_TOCTOU) {

TEST_P(CompilationCachingTest, SaveToCache_TOCTOU) {

    if (!mIsCachingSupported) return;



    // Create test models and check if fully supported by the service.
@@ -1053,7 +1150,7 @@ TEST_F(CompilationCachingTest, SaveToCache_TOCTOU) { 
    }

}



TEST_F(CompilationCachingTest, PrepareFromCache_TOCTOU) {

TEST_P(CompilationCachingTest, PrepareFromCache_TOCTOU) {

    if (!mIsCachingSupported) return;



    // Create test models and check if fully supported by the service.
@@ -1116,7 +1213,7 @@ TEST_F(CompilationCachingTest, PrepareFromCache_TOCTOU) { 
    }

}



TEST_F(CompilationCachingTest, ReplaceSecuritySensitiveCache) {

TEST_P(CompilationCachingTest, ReplaceSecuritySensitiveCache) {

    if (!mIsCachingSupported) return;



    // Create test models and check if fully supported by the service.
@@ -1164,11 +1261,19 @@ TEST_F(CompilationCachingTest, ReplaceSecuritySensitiveCache) { 
    }

}



class CompilationCachingSecurityTest : public CompilationCachingTest,

                                       public ::testing::WithParamInterface<uint32_t> {

static const auto kOperandTypeChoices =

        ::testing::Values(OperandType::TENSOR_FLOAT32, OperandType::TENSOR_QUANT8_ASYMM);



INSTANTIATE_TEST_CASE_P(TestCompilationCaching, CompilationCachingTest, kOperandTypeChoices);



class CompilationCachingSecurityTest

    : public CompilationCachingTestBase,

      public ::testing::WithParamInterface<std::tuple<OperandType, uint32_t>> {

  protected:

    CompilationCachingSecurityTest() : CompilationCachingTestBase(std::get<0>(GetParam())) {}



    void SetUp() {

        CompilationCachingTest::SetUp();

        CompilationCachingTestBase::SetUp();

        generator.seed(kSeed);

    }
@@ -1254,7 +1359,7 @@ class CompilationCachingSecurityTest : public CompilationCachingTest, 
        }

    }



    const uint32_t kSeed = GetParam();

    const uint32_t kSeed = std::get<1>(GetParam());

    std::mt19937 generator;

};
@@ -1302,7 +1407,7 @@ TEST_P(CompilationCachingSecurityTest, WrongToken) { 
}



INSTANTIATE_TEST_CASE_P(TestCompilationCaching, CompilationCachingSecurityTest,

                        ::testing::Range(0U, 10U));

                        ::testing::Combine(kOperandTypeChoices, ::testing::Range(0U, 10U)));



}  // namespace functional

}  // namespace vts