Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 13fb7e4e authored by Yangster's avatar Yangster
Browse files

Statsd cpu optimizaton 

- cache the changed dimensions in condition tracker.
- avoid query condition wizard when unnecessary.
- avoid copy dimension keys in condition key generation.

Test: statsd tests.

BUG: b/73959649
Change-Id: I17d68e2a82643de3f421309841e75f84c6fd8f43
parent c8a6305d

cmds/statsd/Android.mk

+26 −3
Original line number Diff line number Diff line
@@ -199,7 +199,9 @@ LOCAL_SRC_FILES := \ 
    tests/e2e/MetricConditionLink_e2e_test.cpp \

    tests/e2e/Attribution_e2e_test.cpp \

    tests/e2e/GaugeMetric_e2e_test.cpp \

    tests/e2e/DimensionInCondition_e2e_test.cpp

    tests/e2e/DimensionInCondition_e2e_combination_AND_cond_test.cpp \

    tests/e2e/DimensionInCondition_e2e_combination_OR_cond_test.cpp \

    tests/e2e/DimensionInCondition_e2e_simple_cond_test.cpp



LOCAL_STATIC_LIBRARIES := \

    $(statsd_common_static_libraries) \
@@ -247,11 +249,32 @@ include $(CLEAR_VARS) 
LOCAL_MODULE := statsd_benchmark



LOCAL_SRC_FILES := $(statsd_common_src) \

                    src/atom_field_options.proto \

                    src/atoms.proto \

                    src/stats_log.proto \

                   benchmark/main.cpp \

                   benchmark/hello_world_benchmark.cpp \

                   benchmark/log_event_benchmark.cpp \

                   benchmark/stats_write_benchmark.cpp \

                   benchmark/filter_value_benchmark.cpp

                   benchmark/filter_value_benchmark.cpp \

                   benchmark/get_dimensions_for_condition_benchmark.cpp \

                   benchmark/metric_util.cpp \

                   benchmark/duration_metric_benchmark.cpp



LOCAL_STATIC_LIBRARIES := \

    $(statsd_common_static_libraries)



LOCAL_PROTOC_OPTIMIZE_TYPE := full



LOCAL_PROTOC_FLAGS := \

    -Iexternal/protobuf/src



LOCAL_SHARED_LIBRARIES := $(statsd_common_shared_libraries) \

                        libprotobuf-cpp-full





LOCAL_STATIC_JAVA_LIBRARIES := \

    platformprotoslite



LOCAL_C_INCLUDES := $(statsd_common_c_includes)

cmds/statsd/benchmark/duration_metric_benchmark.cpp

0 → 100644
+320 −0
Original line number Diff line number Diff line 
/*

 * Copyright (C) 2018 The Android Open Source Project

 *

 * Licensed under the Apache License, Version 2.0 (the "License");

 * you may not use this file except in compliance with the License.

 * You may obtain a copy of the License at

 *

 *      http://www.apache.org/licenses/LICENSE-2.0

 *

 * Unless required by applicable law or agreed to in writing, software

 * distributed under the License is distributed on an "AS IS" BASIS,

 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

 * See the License for the specific language governing permissions and

 * limitations under the License.

 */

#include <vector>

#include "benchmark/benchmark.h"

#include "FieldValue.h"

#include "HashableDimensionKey.h"

#include "logd/LogEvent.h"

#include "stats_log_util.h"

#include "metric_util.h"



namespace android {

namespace os {

namespace statsd {



using std::vector;



static StatsdConfig CreateDurationMetricConfig_NoLink_AND_CombinationCondition(

        DurationMetric::AggregationType aggregationType, bool addExtraDimensionInCondition) {

    StatsdConfig config;

    *config.add_atom_matcher() = CreateStartScheduledJobAtomMatcher();

    *config.add_atom_matcher() = CreateFinishScheduledJobAtomMatcher();

    *config.add_atom_matcher() = CreateSyncStartAtomMatcher();

    *config.add_atom_matcher() = CreateSyncEndAtomMatcher();

    *config.add_atom_matcher() = CreateScreenTurnedOnAtomMatcher();

    *config.add_atom_matcher() = CreateScreenTurnedOffAtomMatcher();



    auto scheduledJobPredicate = CreateScheduledJobPredicate();

    auto dimensions = scheduledJobPredicate.mutable_simple_predicate()->mutable_dimensions();

    dimensions->set_field(android::util::SCHEDULED_JOB_STATE_CHANGED);

    dimensions->add_child()->set_field(2);  // job name field.



    auto screenIsOffPredicate = CreateScreenIsOffPredicate();



    auto isSyncingPredicate = CreateIsSyncingPredicate();

    auto syncDimension = isSyncingPredicate.mutable_simple_predicate()->mutable_dimensions();

    *syncDimension = CreateAttributionUidAndTagDimensions(android::util::SYNC_STATE_CHANGED,

                                                          {Position::FIRST});

    if (addExtraDimensionInCondition) {

        syncDimension->add_child()->set_field(2 /* name field*/);

    }



    *config.add_predicate() = scheduledJobPredicate;

    *config.add_predicate() = screenIsOffPredicate;

    *config.add_predicate() = isSyncingPredicate;

    auto combinationPredicate = config.add_predicate();

    combinationPredicate->set_id(StringToId("CombinationPredicate"));

    combinationPredicate->mutable_combination()->set_operation(LogicalOperation::AND);

    addPredicateToPredicateCombination(screenIsOffPredicate, combinationPredicate);

    addPredicateToPredicateCombination(isSyncingPredicate, combinationPredicate);



    auto metric = config.add_duration_metric();

    metric->set_bucket(FIVE_MINUTES);

    metric->set_id(StringToId("scheduledJob"));

    metric->set_what(scheduledJobPredicate.id());

    metric->set_condition(combinationPredicate->id());

    metric->set_aggregation_type(aggregationType);

    auto dimensionWhat = metric->mutable_dimensions_in_what();

    dimensionWhat->set_field(android::util::SCHEDULED_JOB_STATE_CHANGED);

    dimensionWhat->add_child()->set_field(2);  // job name field.

    *metric->mutable_dimensions_in_condition() = CreateAttributionUidAndTagDimensions(

            android::util::SYNC_STATE_CHANGED, {Position::FIRST});

    return config;

}



static StatsdConfig CreateDurationMetricConfig_Link_AND_CombinationCondition(

        DurationMetric::AggregationType aggregationType, bool addExtraDimensionInCondition) {

    StatsdConfig config;

    *config.add_atom_matcher() = CreateStartScheduledJobAtomMatcher();

    *config.add_atom_matcher() = CreateFinishScheduledJobAtomMatcher();

    *config.add_atom_matcher() = CreateSyncStartAtomMatcher();

    *config.add_atom_matcher() = CreateSyncEndAtomMatcher();

    *config.add_atom_matcher() = CreateScreenTurnedOnAtomMatcher();

    *config.add_atom_matcher() = CreateScreenTurnedOffAtomMatcher();



    auto scheduledJobPredicate = CreateScheduledJobPredicate();

    auto dimensions = scheduledJobPredicate.mutable_simple_predicate()->mutable_dimensions();

    *dimensions = CreateAttributionUidDimensions(

                android::util::SCHEDULED_JOB_STATE_CHANGED, {Position::FIRST});

    dimensions->add_child()->set_field(2);  // job name field.



    auto isSyncingPredicate = CreateIsSyncingPredicate();

    auto syncDimension = isSyncingPredicate.mutable_simple_predicate()->mutable_dimensions();

    *syncDimension = CreateAttributionUidDimensions(

            android::util::SYNC_STATE_CHANGED, {Position::FIRST});

    if (addExtraDimensionInCondition) {

        syncDimension->add_child()->set_field(2 /* name field*/);

    }



    auto screenIsOffPredicate = CreateScreenIsOffPredicate();



    *config.add_predicate() = scheduledJobPredicate;

    *config.add_predicate() = screenIsOffPredicate;

    *config.add_predicate() = isSyncingPredicate;

    auto combinationPredicate = config.add_predicate();

    combinationPredicate->set_id(StringToId("CombinationPredicate"));

    combinationPredicate->mutable_combination()->set_operation(LogicalOperation::AND);

    addPredicateToPredicateCombination(screenIsOffPredicate, combinationPredicate);

    addPredicateToPredicateCombination(isSyncingPredicate, combinationPredicate);



    auto metric = config.add_duration_metric();

    metric->set_bucket(FIVE_MINUTES);

    metric->set_id(StringToId("scheduledJob"));

    metric->set_what(scheduledJobPredicate.id());

    metric->set_condition(combinationPredicate->id());

    metric->set_aggregation_type(aggregationType);

    *metric->mutable_dimensions_in_what() = CreateAttributionUidDimensions(

            android::util::SCHEDULED_JOB_STATE_CHANGED, {Position::FIRST});



    auto links = metric->add_links();

    links->set_condition(isSyncingPredicate.id());

    *links->mutable_fields_in_what() =

            CreateAttributionUidDimensions(

                android::util::SCHEDULED_JOB_STATE_CHANGED, {Position::FIRST});

    *links->mutable_fields_in_condition() =

            CreateAttributionUidDimensions(android::util::SYNC_STATE_CHANGED, {Position::FIRST});

    return config;

}



static void BM_DurationMetricNoLink(benchmark::State& state) {

    ConfigKey cfgKey;

    auto config = CreateDurationMetricConfig_NoLink_AND_CombinationCondition(

            DurationMetric::SUM, false);

    int64_t bucketStartTimeNs = 10000000000;

    int64_t bucketSizeNs =

            TimeUnitToBucketSizeInMillis(config.duration_metric(0).bucket()) * 1000000LL;





    std::vector<AttributionNodeInternal> attributions1 = {

            CreateAttribution(111, "App1"), CreateAttribution(222, "GMSCoreModule1"),

            CreateAttribution(222, "GMSCoreModule2")};



    std::vector<AttributionNodeInternal> attributions2 = {

            CreateAttribution(333, "App2"), CreateAttribution(222, "GMSCoreModule1"),

            CreateAttribution(555, "GMSCoreModule2")};



    std::vector<std::unique_ptr<LogEvent>> events;



    events.push_back(CreateScreenStateChangedEvent(android::view::DISPLAY_STATE_OFF,

                                                   bucketStartTimeNs + 11));

    events.push_back(CreateScreenStateChangedEvent(android::view::DISPLAY_STATE_ON,

                                                   bucketStartTimeNs + 40));



    events.push_back(CreateScreenStateChangedEvent(android::view::DISPLAY_STATE_OFF,

                                                   bucketStartTimeNs + 102));

    events.push_back(CreateScreenStateChangedEvent(android::view::DISPLAY_STATE_ON,

                                                   bucketStartTimeNs + 450));



    events.push_back(CreateScreenStateChangedEvent(android::view::DISPLAY_STATE_OFF,

                                                   bucketStartTimeNs + 650));

    events.push_back(CreateScreenStateChangedEvent(android::view::DISPLAY_STATE_ON,

                                                   bucketStartTimeNs + bucketSizeNs + 100));



    events.push_back(CreateScreenStateChangedEvent(android::view::DISPLAY_STATE_OFF,

                                                   bucketStartTimeNs + bucketSizeNs + 640));

    events.push_back(CreateScreenStateChangedEvent(android::view::DISPLAY_STATE_ON,

                                                   bucketStartTimeNs + bucketSizeNs + 650));



    events.push_back(CreateStartScheduledJobEvent(

            {CreateAttribution(9999, "")}, "job0", bucketStartTimeNs + 2));

    events.push_back(CreateFinishScheduledJobEvent(

            {CreateAttribution(9999, "")}, "job0",bucketStartTimeNs + 101));



    events.push_back(CreateStartScheduledJobEvent(

            {CreateAttribution(9999, "")}, "job2", bucketStartTimeNs + 201));

    events.push_back(CreateFinishScheduledJobEvent(

            {CreateAttribution(9999, "")}, "job2",bucketStartTimeNs + 500));



    events.push_back(CreateStartScheduledJobEvent(

            {CreateAttribution(8888, "")}, "job2", bucketStartTimeNs + 600));

    events.push_back(CreateFinishScheduledJobEvent(

            {CreateAttribution(8888, "")}, "job2",bucketStartTimeNs + bucketSizeNs + 850));



    events.push_back(CreateStartScheduledJobEvent(

            {CreateAttribution(8888, "")}, "job1", bucketStartTimeNs + bucketSizeNs + 600));

    events.push_back(CreateFinishScheduledJobEvent(

            {CreateAttribution(8888, "")}, "job1", bucketStartTimeNs + bucketSizeNs + 900));



    events.push_back(CreateSyncStartEvent(attributions1, "ReadEmail",

                                          bucketStartTimeNs + 10));

    events.push_back(CreateSyncEndEvent(attributions1, "ReadEmail",

                                        bucketStartTimeNs + 50));



    events.push_back(CreateSyncStartEvent(attributions1, "ReadEmail",

                                          bucketStartTimeNs + 200));

    events.push_back(CreateSyncEndEvent(attributions1, "ReadEmail",

                                        bucketStartTimeNs + bucketSizeNs + 300));



    events.push_back(CreateSyncStartEvent(attributions1, "ReadDoc",

                                          bucketStartTimeNs + 400));

    events.push_back(CreateSyncEndEvent(attributions1, "ReadDoc",

                                        bucketStartTimeNs + bucketSizeNs - 1));



    events.push_back(CreateSyncStartEvent(attributions2, "ReadEmail",

                                          bucketStartTimeNs + 401));

    events.push_back(CreateSyncEndEvent(attributions2, "ReadEmail",

                                        bucketStartTimeNs + bucketSizeNs + 700));



    sortLogEventsByTimestamp(&events);



    while (state.KeepRunning()) {

        auto processor = CreateStatsLogProcessor(

                bucketStartTimeNs / NS_PER_SEC, config, cfgKey);

        for (const auto& event : events) {

            processor->OnLogEvent(event.get());

        }

    }

}



BENCHMARK(BM_DurationMetricNoLink);





static void BM_DurationMetricLink(benchmark::State& state) {

    ConfigKey cfgKey;

    auto config = CreateDurationMetricConfig_Link_AND_CombinationCondition(

        DurationMetric::SUM, false);

    int64_t bucketStartTimeNs = 10000000000;

    int64_t bucketSizeNs =

            TimeUnitToBucketSizeInMillis(config.duration_metric(0).bucket()) * 1000000LL;



    std::vector<AttributionNodeInternal> attributions1 = {

            CreateAttribution(111, "App1"), CreateAttribution(222, "GMSCoreModule1"),

            CreateAttribution(222, "GMSCoreModule2")};



    std::vector<AttributionNodeInternal> attributions2 = {

            CreateAttribution(333, "App2"), CreateAttribution(222, "GMSCoreModule1"),

            CreateAttribution(555, "GMSCoreModule2")};



    std::vector<AttributionNodeInternal> attributions3 = {

            CreateAttribution(444, "App3"), CreateAttribution(222, "GMSCoreModule1"),

            CreateAttribution(555, "GMSCoreModule2")};



    std::vector<std::unique_ptr<LogEvent>> events;



    events.push_back(CreateScreenStateChangedEvent(android::view::DISPLAY_STATE_OFF,

                                                   bucketStartTimeNs + 55));

    events.push_back(CreateScreenStateChangedEvent(android::view::DISPLAY_STATE_ON,

                                                   bucketStartTimeNs + 120));

    events.push_back(CreateScreenStateChangedEvent(android::view::DISPLAY_STATE_OFF,

                                                   bucketStartTimeNs + 121));

    events.push_back(CreateScreenStateChangedEvent(android::view::DISPLAY_STATE_ON,

                                                   bucketStartTimeNs + 450));



    events.push_back(CreateScreenStateChangedEvent(android::view::DISPLAY_STATE_OFF,

                                                   bucketStartTimeNs + 501));

    events.push_back(CreateScreenStateChangedEvent(android::view::DISPLAY_STATE_ON,

                                                   bucketStartTimeNs + bucketSizeNs + 100));



    events.push_back(CreateStartScheduledJobEvent(

            {CreateAttribution(111, "App1")}, "job1", bucketStartTimeNs + 1));

    events.push_back(CreateFinishScheduledJobEvent(

            {CreateAttribution(111, "App1")}, "job1",bucketStartTimeNs + 101));



    events.push_back(CreateStartScheduledJobEvent(

            {CreateAttribution(333, "App2")}, "job2", bucketStartTimeNs + 201));

    events.push_back(CreateFinishScheduledJobEvent(

            {CreateAttribution(333, "App2")}, "job2",bucketStartTimeNs + 500));

    events.push_back(CreateStartScheduledJobEvent(

            {CreateAttribution(333, "App2")}, "job2", bucketStartTimeNs + 600));

    events.push_back(CreateFinishScheduledJobEvent(

            {CreateAttribution(333, "App2")}, "job2",

            bucketStartTimeNs + bucketSizeNs + 850));



    events.push_back(

        CreateStartScheduledJobEvent({CreateAttribution(444, "App3")}, "job3",

                                     bucketStartTimeNs + bucketSizeNs - 2));

    events.push_back(

        CreateFinishScheduledJobEvent({CreateAttribution(444, "App3")}, "job3",

                                      bucketStartTimeNs + bucketSizeNs + 900));



    events.push_back(CreateSyncStartEvent(attributions1, "ReadEmail",

                                          bucketStartTimeNs + 50));

    events.push_back(CreateSyncEndEvent(attributions1, "ReadEmail",

                                        bucketStartTimeNs + 110));



    events.push_back(CreateSyncStartEvent(attributions2, "ReadEmail",

                                          bucketStartTimeNs + 300));

    events.push_back(CreateSyncEndEvent(attributions2, "ReadEmail",

                                        bucketStartTimeNs + bucketSizeNs + 700));

    events.push_back(CreateSyncStartEvent(attributions2, "ReadDoc",

                                          bucketStartTimeNs + 400));

    events.push_back(CreateSyncEndEvent(attributions2, "ReadDoc",

                                        bucketStartTimeNs + bucketSizeNs - 1));



    events.push_back(CreateSyncStartEvent(attributions3, "ReadDoc",

                                          bucketStartTimeNs + 550));

    events.push_back(CreateSyncEndEvent(attributions3, "ReadDoc",

                                        bucketStartTimeNs + 800));

    events.push_back(CreateSyncStartEvent(attributions3, "ReadDoc",

                                          bucketStartTimeNs + bucketSizeNs - 1));

    events.push_back(CreateSyncEndEvent(attributions3, "ReadDoc",

                                        bucketStartTimeNs + bucketSizeNs + 700));

    sortLogEventsByTimestamp(&events);



    while (state.KeepRunning()) {

        auto processor = CreateStatsLogProcessor(

                bucketStartTimeNs / NS_PER_SEC, config, cfgKey);

        for (const auto& event : events) {

            processor->OnLogEvent(event.get());

        }

    }

}



BENCHMARK(BM_DurationMetricLink);



}  //  namespace statsd

}  //  namespace os

}  //  namespace android

cmds/statsd/benchmark/filter_value_benchmark.cpp

+37 −8
Original line number Diff line number Diff line
@@ -18,6 +18,7 @@ 
#include "FieldValue.h"

#include "HashableDimensionKey.h"

#include "logd/LogEvent.h"

#include "stats_log_util.h"



namespace android {

namespace os {
@@ -25,17 +26,29 @@ namespace statsd { 


using std::vector;



static void createLogEventAndMatcher(LogEvent* event, FieldMatcher *field_matcher) {

    AttributionNodeInternal node;

    node.set_uid(100);

    node.set_tag("LOCATION");



    std::vector<AttributionNodeInternal> nodes = {node, node};

    event->write(nodes);

    event->write(3.2f);

    event->write("LOCATION");

    event->write((int64_t)990);

    event->init();



    field_matcher->set_field(1);

    auto child = field_matcher->add_child();

    child->set_field(1);

    child->set_position(FIRST);

    child->add_child()->set_field(1);

}



static void BM_FilterValue(benchmark::State& state) {

    LogEvent event(1, 100000);

    event.write(3.2f);

    event.write("LOCATION");

    event.write((int64_t)990);

    event.init();



    FieldMatcher field_matcher;

    field_matcher.set_field(1);

    field_matcher.add_child()->set_field(2);

    field_matcher.add_child()->set_field(3);

    createLogEventAndMatcher(&event, &field_matcher);



    std::vector<Matcher> matchers;

    translateFieldMatcher(field_matcher, &matchers);
@@ -48,6 +61,22 @@ static void BM_FilterValue(benchmark::State& state) { 


BENCHMARK(BM_FilterValue);



static void BM_FilterValue2(benchmark::State& state) {

    LogEvent event(1, 100000);

    FieldMatcher field_matcher;

    createLogEventAndMatcher(&event, &field_matcher);



    std::vector<Matcher> matchers;

    translateFieldMatcher(field_matcher, &matchers);



    while (state.KeepRunning()) {

        HashableDimensionKey output;

        filterValues(matchers, event.getValues(), &output);

    }

}



BENCHMARK(BM_FilterValue2);



}  //  namespace statsd

}  //  namespace os

}  //  namespace android

cmds/statsd/benchmark/get_dimensions_for_condition_benchmark.cpp

0 → 100644
+71 −0
Original line number Diff line number Diff line 
/*

 * Copyright (C) 2018 The Android Open Source Project

 *

 * Licensed under the Apache License, Version 2.0 (the "License");

 * you may not use this file except in compliance with the License.

 * You may obtain a copy of the License at

 *

 *      http://www.apache.org/licenses/LICENSE-2.0

 *

 * Unless required by applicable law or agreed to in writing, software

 * distributed under the License is distributed on an "AS IS" BASIS,

 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

 * See the License for the specific language governing permissions and

 * limitations under the License.

 */

#include <vector>

#include "benchmark/benchmark.h"

#include "FieldValue.h"

#include "HashableDimensionKey.h"

#include "logd/LogEvent.h"

#include "stats_log_util.h"



namespace android {

namespace os {

namespace statsd {



using std::vector;



static void createLogEventAndLink(LogEvent* event, Metric2Condition *link) {

    AttributionNodeInternal node;

    node.set_uid(100);

    node.set_tag("LOCATION");



    std::vector<AttributionNodeInternal> nodes = {node, node};

    event->write(nodes);

    event->write(3.2f);

    event->write("LOCATION");

    event->write((int64_t)990);

    event->init();



    link->conditionId = 1;



    FieldMatcher field_matcher;

    field_matcher.set_field(event->GetTagId());

    auto child = field_matcher.add_child();

    child->set_field(1);

    child->set_position(FIRST);

    child->add_child()->set_field(1);



    translateFieldMatcher(field_matcher, &link->metricFields);

    field_matcher.set_field(event->GetTagId() + 1);

    translateFieldMatcher(field_matcher, &link->conditionFields);

}



static void BM_GetDimensionInCondition(benchmark::State& state) {

    Metric2Condition link;

    LogEvent event(1, 100000);

    createLogEventAndLink(&event, &link);



    while (state.KeepRunning()) {

        HashableDimensionKey output;

        getDimensionForCondition(event.getValues(), link, &output);

    }

}



BENCHMARK(BM_GetDimensionInCondition);





}  //  namespace statsd

}  //  namespace os

}  //  namespace android

cmds/statsd/benchmark/metric_util.cpp

0 → 100644
+394 −0

File added.

Preview size limit exceeded, changes collapsed.