Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit caf339d0 authored by Yao Chen's avatar Yao Chen
Browse files

More complete implementation for condition and log matchers in statsd. 

+ also synced proto from google3 to fix the LogEntryMatcher proto

+ MetricsManager represents StatsdConfig, it's responsible for initializing and managing all
LogEntryMatcher, Condition, and Metrics. Start review from here.

+ Added more complete StatsdConfig initialization, including building the map for:
    LogEntryMatcher -> Metrics
    LogEntryMatcher -> Condition
    Condition       -> Metrics.

    All the maps use index(int). The extra amount of memory for storing mappings help us
    quickly process log events.

  The StatsdConfig initialization process detects malformed config
  - Circle dependency
  - Missing definition
  etc.

 And once we detect ANY error, statsd will reject the config. And the resources related to this
 config will be released.

Test: Added unit tests
Change-Id: I2c4aefdbf3e2aa1701eacbb2fb5e653819ec1fbb
parent 077734be

cmds/statsd/Android.mk

+12 −23
Original line number Diff line number Diff line
@@ -42,24 +42,7 @@ LOCAL_MODULE := statsd 
LOCAL_SRC_FILES := \

    ../../core/java/android/os/IStatsCompanionService.aidl \

    ../../core/java/android/os/IStatsManager.aidl \

    src/StatsService.cpp \

    src/AnomalyMonitor.cpp \

    src/LogEntryPrinter.cpp \

    src/LogReader.cpp \

    src/main.cpp \

    src/DropboxWriter.cpp \

    src/parse_util.cpp \

    src/StatsLogProcessor.cpp \

    src/stats_log.proto \

    src/statsd_config.proto \

    src/StatsPullerManager.cpp \

    src/KernelWakelockPuller.cpp \

    src/DropboxReader.cpp \

    src/matchers/LogEntryMatcherManager.cpp \

    src/metrics/CountMetricProducer.cpp \

    src/metrics/ConditionTracker.cpp \

    src/metrics/MetricsManager.cpp \

    src/metrics/CountAnomalyTracker.cpp \

    $(call all-cpp-files-under,src) \



LOCAL_CFLAGS += \

    -Wall \
@@ -129,13 +112,19 @@ LOCAL_SRC_FILES := \ 
    ../../core/java/android/os/IStatsCompanionService.aidl \

    ../../core/java/android/os/IStatsManager.aidl \

    src/StatsService.cpp \

    tests/indexed_priority_queue_test.cpp \

    src/parse_util.cpp \

    src/stats_util.cpp \

    src/LogEntryPrinter.cpp \

    src/LogReader.cpp \

    src/matchers/LogEntryMatcherManager.cpp \

    tests/LogReader_test.cpp \

    tests/LogEntryMatcher_test.cpp \

    src/matchers/matcher_util.cpp \

    src/condition/SimpleConditionTracker.cpp \

    src/condition/CombinationConditionTracker.cpp \

    src/matchers/SimpleLogMatchingTracker.cpp \

    src/matchers/CombinationLogMatchingTracker.cpp \

    src/metrics/metrics_manager_util.cpp \

    src/metrics/CountMetricProducer.cpp \

    src/metrics/CountAnomalyTracker.cpp \

    src/condition/condition_util.cpp \

    $(call all-cpp-files-under, tests) \



LOCAL_STATIC_LIBRARIES := \

    libgmock \

cmds/statsd/src/StatsLogProcessor.cpp

+8 −24
Original line number Diff line number Diff line
@@ -20,7 +20,6 @@ 
#include <frameworks/base/cmds/statsd/src/stats_log.pb.h>

#include <log/log_event_list.h>

#include <metrics/CountMetricProducer.h>

#include <parse_util.h>

#include <utils/Errors.h>



using namespace android;
@@ -41,28 +40,6 @@ StatsLogProcessor::StatsLogProcessor() : m_dropbox_writer("all-logs") { 
StatsLogProcessor::~StatsLogProcessor() {

}



StatsdConfig StatsLogProcessor::buildFakeConfig() {

    // HACK: Hard code a test metric for counting screen on events...

    StatsdConfig config;

    config.set_config_id(12345L);



    CountMetric* metric = config.add_count_metric();

    metric->set_metric_id(20150717L);

    metric->set_what("SCREEN_IS_ON");

    metric->mutable_bucket()->set_bucket_size_millis(30 * 1000L);



    LogEntryMatcher* eventMatcher = config.add_log_entry_matcher();

    eventMatcher->set_name("SCREEN_IS_ON");



    SimpleLogEntryMatcher* simpleLogEntryMatcher = eventMatcher->mutable_simple_log_entry_matcher();

    simpleLogEntryMatcher->add_tag(2 /*SCREEN_STATE_CHANGE*/);

    simpleLogEntryMatcher->add_key_value_matcher()->mutable_key_matcher()

            ->set_key(1 /*SCREEN_STATE_CHANGE__DISPLAY_STATE*/);

    simpleLogEntryMatcher->mutable_key_value_matcher(0)

            ->set_eq_int(2/*SCREEN_STATE_CHANGE__DISPLAY_STATE__STATE_ON*/);

    return config;

}



// TODO: what if statsd service restarts? How do we know what logs are already processed before?

void StatsLogProcessor::OnLogEvent(const log_msg& msg) {

    // TODO: Use EventMetric to filter the events we want to log.
@@ -83,7 +60,14 @@ void StatsLogProcessor::UpdateConfig(const int config_source, const StatsdConfig 


    ALOGD("Updated configuration for source %i", config_source);



    mMetricsManagers.insert({config_source, std::make_unique<MetricsManager>(config)});

    unique_ptr<MetricsManager> newMetricsManager = std::make_unique<MetricsManager>(config);

    if (newMetricsManager->isConfigValid()) {

        mMetricsManagers.insert({config_source, std::move(newMetricsManager)});

        ALOGD("StatsdConfig valid");

    } else {

        // If there is any error in the config, don't use it.

        ALOGD("StatsdConfig NOT valid");

    }

}



}  // namespace statsd

cmds/statsd/src/StatsLogProcessor.h

+2 −4
Original line number Diff line number Diff line
@@ -16,11 +16,11 @@ 
#ifndef STATS_LOG_PROCESSOR_H

#define STATS_LOG_PROCESSOR_H



#include "frameworks/base/cmds/statsd/src/statsd_config.pb.h"

#include "DropboxWriter.h"

#include "LogReader.h"

#include "frameworks/base/cmds/statsd/src/statsd_config.pb.h"

#include "metrics/MetricsManager.h"

#include "parse_util.h"

#include "stats_util.h"



#include <log/logprint.h>

#include <stdio.h>
@@ -44,8 +44,6 @@ private: 
    DropboxWriter m_dropbox_writer;



    std::unordered_map<int, std::unique_ptr<MetricsManager>> mMetricsManagers;



    static StatsdConfig buildFakeConfig();

};



}  // namespace statsd

cmds/statsd/src/condition/CombinationConditionTracker.cpp

0 → 100644
+136 −0
Original line number Diff line number Diff line 
/*

 * Copyright (C) 2017 The Android Open Source Project

 *

 * Licensed under the Apache License, Version 2.0 (the "License");

 * you may not use this file except in compliance with the License.

 * You may obtain a copy of the License at

 *

 *      http://www.apache.org/licenses/LICENSE-2.0

 *

 * Unless required by applicable law or agreed to in writing, software

 * distributed under the License is distributed on an "AS IS" BASIS,

 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

 * See the License for the specific language governing permissions and

 * limitations under the License.

 */

#define LOG_TAG "CombinationConditionTracker"

#define DEBUG true  // STOPSHIP if true

#define VLOG(...) \

    if (DEBUG) ALOGD(__VA_ARGS__);



#include "CombinationConditionTracker.h"

#include <cutils/log.h>

#include <log/logprint.h>

using std::string;

using std::unique_ptr;

using std::unordered_map;

using std::vector;



namespace android {

namespace os {

namespace statsd {



CombinationConditionTracker::CombinationConditionTracker(const string& name, const int index)

    : ConditionTracker(name, index) {

    VLOG("creating CombinationConditionTracker %s", mName.c_str());

}



CombinationConditionTracker::~CombinationConditionTracker() {

    VLOG("~CombinationConditionTracker() %s", mName.c_str());

}



bool CombinationConditionTracker::init(const vector<Condition>& allConditionConfig,

                                       const vector<sp<ConditionTracker>>& allConditionTrackers,

                                       const unordered_map<string, int>& conditionNameIndexMap,

                                       vector<bool>& stack) {

    VLOG("Combiniation condition init() %s", mName.c_str());

    if (mInitialized) {

        return true;

    }



    // mark this node as visited in the recursion stack.

    stack[mIndex] = true;



    Condition_Combination combinationCondition = allConditionConfig[mIndex].combination();



    if (!combinationCondition.has_operation()) {

        return false;

    }

    mLogicalOperation = combinationCondition.operation();



    if (mLogicalOperation == LogicalOperation::NOT && combinationCondition.condition_size() != 1) {

        return false;

    }



    for (string child : combinationCondition.condition()) {

        auto it = conditionNameIndexMap.find(child);



        if (it == conditionNameIndexMap.end()) {

            ALOGW("Condition %s not found in the config", child.c_str());

            return false;

        }



        int childIndex = it->second;

        const auto& childTracker = allConditionTrackers[childIndex];

        // if the child is a visited node in the recursion -> circle detected.

        if (stack[childIndex]) {

            ALOGW("Circle detected!!!");

            return false;

        }



        bool initChildSucceeded = childTracker->init(allConditionConfig, allConditionTrackers,

                                                     conditionNameIndexMap, stack);



        if (!initChildSucceeded) {

            ALOGW("Child initialization failed %s ", child.c_str());

            return false;

        } else {

            ALOGW("Child initialization success %s ", child.c_str());

        }



        mChildren.push_back(childIndex);



        mTrackerIndex.insert(childTracker->getLogTrackerIndex().begin(),

                             childTracker->getLogTrackerIndex().end());

    }



    // unmark this node in the recursion stack.

    stack[mIndex] = false;



    mInitialized = true;



    return true;

}



bool CombinationConditionTracker::evaluateCondition(

        const LogEventWrapper& event, const std::vector<MatchingState>& eventMatcherValues,

        const std::vector<sp<ConditionTracker>>& mAllConditions,

        std::vector<ConditionState>& conditionCache, std::vector<bool>& changedCache) {

    // value is up to date.

    if (conditionCache[mIndex] != ConditionState::kNotEvaluated) {

        return false;

    }



    for (const int childIndex : mChildren) {

        if (conditionCache[childIndex] == ConditionState::kNotEvaluated) {

            const sp<ConditionTracker>& child = mAllConditions[childIndex];

            child->evaluateCondition(event, eventMatcherValues, mAllConditions, conditionCache,

                                     changedCache);

        }

    }



    ConditionState newCondition =

            evaluateCombinationCondition(mChildren, mLogicalOperation, conditionCache);



    bool changed = (mConditionState != newCondition);

    mConditionState = newCondition;



    conditionCache[mIndex] = mConditionState;



    changedCache[mIndex] = changed;

    return changed;

}



}  // namespace statsd

}  // namespace os

}  // namespace android

cmds/statsd/src/condition/CombinationConditionTracker.h

0 → 100644
+57 −0
Original line number Diff line number Diff line 
/*

 * Copyright (C) 2017 The Android Open Source Project

 *

 * Licensed under the Apache License, Version 2.0 (the "License");

 * you may not use this file except in compliance with the License.

 * You may obtain a copy of the License at

 *

 *      http://www.apache.org/licenses/LICENSE-2.0

 *

 * Unless required by applicable law or agreed to in writing, software

 * distributed under the License is distributed on an "AS IS" BASIS,

 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

 * See the License for the specific language governing permissions and

 * limitations under the License.

 */



#ifndef COMBINATION_CONDITION_TRACKER_H

#define COMBINATION_CONDITION_TRACKER_H



#include "ConditionTracker.h"

#include "frameworks/base/cmds/statsd/src/statsd_config.pb.h"



namespace android {

namespace os {

namespace statsd {



class CombinationConditionTracker : public virtual ConditionTracker {

public:

    CombinationConditionTracker(const std::string& name, const int index);



    ~CombinationConditionTracker();



    bool init(const std::vector<Condition>& allConditionConfig,

              const std::vector<sp<ConditionTracker>>& allConditionTrackers,

              const std::unordered_map<std::string, int>& conditionNameIndexMap,

              std::vector<bool>& stack) override;



    bool evaluateCondition(const LogEventWrapper& event,

                           const std::vector<MatchingState>& eventMatcherValues,

                           const std::vector<sp<ConditionTracker>>& mAllConditions,

                           std::vector<ConditionState>& conditionCache,

                           std::vector<bool>& changedCache) override;



private:

    LogicalOperation mLogicalOperation;

    // Store index of the children Conditions.

    // We don't store string name of the Children, because we want to get rid of the hash map to

    // map the name to object. We don't want to store smart pointers to children, because it

    // increases the risk of circular dependency and memory leak.

    std::vector<int> mChildren;

};



}  // namespace statsd

}  // namespace os

}  // namespace android



#endif  // COMBINATION_CONDITION_TRACKER_H