Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit d9269e2e authored by David Chen's avatar David Chen
Browse files

Adds rate limit to checking byte size. 

Since there is a separate guardrail for memory used by uid map, we
no longer add the memory from uid map with the memory per each
config's metrics. We also prevent the byte size check from happening
too frequently. In order to mock the MetricsManager, we refactor
some of the existing methods.

Test: Added unit-tests and verified they all pass on marlin.
Change-Id: I15cf105f7d95f4016fdb0443b0a33eebe862cafb
parent f10f83d5

cmds/statsd/Android.mk

+1 −0
Original line number Original line Diff line number Diff line
@@ -161,6 +161,7 @@ LOCAL_SRC_FILES := \ 
    tests/LogEntryMatcher_test.cpp \

    tests/LogEntryMatcher_test.cpp \

    tests/LogReader_test.cpp \

    tests/LogReader_test.cpp \

    tests/MetricsManager_test.cpp \

    tests/MetricsManager_test.cpp \

    tests/StatsLogProcessor_test.cpp \

    tests/UidMap_test.cpp \

    tests/UidMap_test.cpp \

    tests/condition/CombinationConditionTracker_test.cpp \

    tests/condition/CombinationConditionTracker_test.cpp \

    tests/condition/SimpleConditionTracker_test.cpp \

    tests/condition/SimpleConditionTracker_test.cpp \

cmds/statsd/src/StatsLogProcessor.cpp

+22 −15
Original line number Original line Diff line number Diff line
@@ -86,9 +86,7 @@ void StatsLogProcessor::OnLogEvent(const LogEvent& msg) { 
    // pass the event to metrics managers.

    // pass the event to metrics managers.

    for (auto& pair : mMetricsManagers) {

    for (auto& pair : mMetricsManagers) {

        pair.second->onLogEvent(msg);

        pair.second->onLogEvent(msg);

        // TODO: THIS CHECK FAILS BECAUSE ONCE UIDMAP SIZE EXCEEDS LIMIT, DROPPING METRICS DATA

        flushIfNecessary(msg.GetTimestampNs(), pair.first, *(pair.second));

        // DOESN'T HELP. FIX THIS.

        // flushIfNecessary(msg.GetTimestampNs(), pair.first, pair.second);

    }

    }





    // Hard-coded logic to update the isolated uid's in the uid-map.

    // Hard-coded logic to update the isolated uid's in the uid-map.
@@ -217,23 +215,32 @@ void StatsLogProcessor::OnConfigRemoved(const ConfigKey& key) { 
    mLastBroadcastTimes.erase(key);

    mLastBroadcastTimes.erase(key);

}

}





void StatsLogProcessor::flushIfNecessary(uint64_t timestampNs,

void StatsLogProcessor::flushIfNecessary(uint64_t timestampNs, const ConfigKey& key,

                                         const ConfigKey& key,

                                         MetricsManager& metricsManager) {

                                         const unique_ptr<MetricsManager>& metricsManager) {

    std::lock_guard<std::mutex> lock(mBroadcastTimesMutex);

    std::lock_guard<std::mutex> lock(mBroadcastTimesMutex);





    size_t totalBytes = metricsManager->byteSize() + mUidMap->getBytesUsed();

    auto lastCheckTime = mLastByteSizeTimes.find(key);

    // TODO: Find a way to test that the dropping and broadcasts are sent when memory is exceeded.

    if (lastCheckTime != mLastByteSizeTimes.end()) {

    if (totalBytes > kMaxSerializedBytes) {  // Too late. We need to start clearing data.

        if (timestampNs - lastCheckTime->second < StatsdStats::kMinByteSizeCheckPeriodNs) {

            return;

        }

    }



    // We suspect that the byteSize() computation is expensive, so we set a rate limit.

    size_t totalBytes = metricsManager.byteSize();

    mLastByteSizeTimes[key] = timestampNs;

    if (totalBytes >

        StatsdStats::kMaxMetricsBytesPerConfig) {  // Too late. We need to start clearing data.

        // We ignore the return value so we force each metric producer to clear its contents.

        // We ignore the return value so we force each metric producer to clear its contents.

        metricsManager->onDumpReport();

        metricsManager.onDumpReport();

        StatsdStats::getInstance().noteDataDropped(key);

        StatsdStats::getInstance().noteDataDropped(key);

        VLOG("StatsD had to toss out metrics for %s", key.ToString().c_str());

        VLOG("StatsD had to toss out metrics for %s", key.ToString().c_str());

    } else if (totalBytes >

    } else if (totalBytes > .9 * StatsdStats::kMaxMetricsBytesPerConfig) {

               .9 * kMaxSerializedBytes) {  // Send broadcast so that receivers can pull data.

        // Send broadcast so that receivers can pull data.

        auto lastFlushNs = mLastBroadcastTimes.find(key);

        auto lastBroadcastTime = mLastBroadcastTimes.find(key);

        if (lastFlushNs != mLastBroadcastTimes.end()) {

        if (lastBroadcastTime != mLastBroadcastTimes.end()) {

            if (timestampNs - lastFlushNs->second < kMinBroadcastPeriod) {

            if (timestampNs - lastBroadcastTime->second < StatsdStats::kMinBroadcastPeriodNs) {

                VLOG("StatsD would've sent a broadcast but the rate limit stopped us.");

                return;

                return;

            }

            }

        }

        }

cmds/statsd/src/StatsLogProcessor.h

+8 −12
Original line number Original line Diff line number Diff line
@@ -60,29 +60,25 @@ private: 




    std::unordered_map<ConfigKey, long> mLastBroadcastTimes;

    std::unordered_map<ConfigKey, long> mLastBroadcastTimes;





    // Tracks when we last checked the bytes consumed for each config key.

    std::unordered_map<ConfigKey, long> mLastByteSizeTimes;



    sp<UidMap> mUidMap;  // Reference to the UidMap to lookup app name and version for each uid.

    sp<UidMap> mUidMap;  // Reference to the UidMap to lookup app name and version for each uid.





    sp<AnomalyMonitor> mAnomalyMonitor;

    sp<AnomalyMonitor> mAnomalyMonitor;





    /* Max *serialized* size of the logs kept in memory before flushing through binder call.

       Proto lite does not implement the SpaceUsed() function which gives the in memory byte size.

       So we cap memory usage by limiting the serialized size. Note that protobuf's in memory size

       is higher than its serialized size.

     */

    static const size_t kMaxSerializedBytes = 16 * 1024;



    /* Check if we should send a broadcast if approaching memory limits and if we're over, we

    /* Check if we should send a broadcast if approaching memory limits and if we're over, we

     * actually delete the data. */

     * actually delete the data. */

    void flushIfNecessary(uint64_t timestampNs,

    void flushIfNecessary(uint64_t timestampNs, const ConfigKey& key,

                          const ConfigKey& key,

                          MetricsManager& metricsManager);

                          const unique_ptr<MetricsManager>& metricsManager);





    // Function used to send a broadcast so that receiver for the config key can call getData

    // Function used to send a broadcast so that receiver for the config key can call getData

    // to retrieve the stored data.

    // to retrieve the stored data.

    std::function<void(const ConfigKey& key)> mSendBroadcast;

    std::function<void(const ConfigKey& key)> mSendBroadcast;





    /* Minimum period between two broadcasts in nanoseconds. Currently set to 60 seconds. */

    FRIEND_TEST(StatsLogProcessorTest, TestRateLimitByteSize);

    static const unsigned long long kMinBroadcastPeriod = 60 * NS_PER_SEC;

    FRIEND_TEST(StatsLogProcessorTest, TestRateLimitBroadcast);

    FRIEND_TEST(StatsLogProcessorTest, TestDropWhenByteSizeTooLarge);

};

};





}  // namespace statsd

}  // namespace statsd

cmds/statsd/src/guardrail/StatsdStats.h

+11 −0
Original line number Original line Diff line number Diff line
@@ -19,6 +19,7 @@ 
#include "frameworks/base/cmds/statsd/src/stats_log.pb.h"

#include "frameworks/base/cmds/statsd/src/stats_log.pb.h"





#include <gtest/gtest_prod.h>

#include <gtest/gtest_prod.h>

#include <log/log_time.h>

#include <mutex>

#include <mutex>

#include <string>

#include <string>

#include <vector>

#include <vector>
@@ -45,10 +46,20 @@ public: 




    const static int kMaxTimestampCount = 20;

    const static int kMaxTimestampCount = 20;





    // Max memory allowed for storing metrics per configuration. When this limit is approached,

    // statsd will send a broadcast so that the client can fetch the data and clear this memory.

    static const size_t kMaxMetricsBytesPerConfig = 128 * 1024;



    // Cap the UID map's memory usage to this. This should be fairly high since the UID information

    // Cap the UID map's memory usage to this. This should be fairly high since the UID information

    // is critical for understanding the metrics.

    // is critical for understanding the metrics.

    const static size_t kMaxBytesUsedUidMap = 50 * 1024;

    const static size_t kMaxBytesUsedUidMap = 50 * 1024;





    /* Minimum period between two broadcasts in nanoseconds. */

    static const unsigned long long kMinBroadcastPeriodNs = 60 * NS_PER_SEC;



    /* Min period between two checks of byte size per config key in nanoseconds. */

    static const unsigned long long kMinByteSizeCheckPeriodNs = 10 * NS_PER_SEC;



    /**

    /**

     * Report a new config has been received and report the static stats about the config.

     * Report a new config has been received and report the static stats about the config.

     *

     *

cmds/statsd/src/metrics/MetricsManager.h

+3 −3
Original line number Original line Diff line number Diff line
@@ -36,7 +36,7 @@ class MetricsManager { 
public:

public:

    MetricsManager(const ConfigKey& configKey, const StatsdConfig& config);

    MetricsManager(const ConfigKey& configKey, const StatsdConfig& config);





    ~MetricsManager();

    virtual ~MetricsManager();





    // Return whether the configuration is valid.

    // Return whether the configuration is valid.

    bool isConfigValid() const;

    bool isConfigValid() const;
@@ -52,11 +52,11 @@ public: 
    void setAnomalyMonitor(const sp<AnomalyMonitor>& anomalyMonitor);

    void setAnomalyMonitor(const sp<AnomalyMonitor>& anomalyMonitor);





    // Config source owner can call onDumpReport() to get all the metrics collected.

    // Config source owner can call onDumpReport() to get all the metrics collected.

    std::vector<std::unique_ptr<std::vector<uint8_t>>> onDumpReport();

    virtual std::vector<std::unique_ptr<std::vector<uint8_t>>> onDumpReport();





    // Computes the total byte size of all metrics managed by a single config source.

    // Computes the total byte size of all metrics managed by a single config source.

    // Does not change the state.

    // Does not change the state.

    size_t byteSize();

    virtual size_t byteSize();





private:

private:

    const ConfigKey mConfigKey;

    const ConfigKey mConfigKey;