Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit c299b2aa authored by Ivo Kay's avatar Ivo Kay
Browse files

Accumulate NetworkStats since boot in StatsPullAtomService 

BytesTransfer atoms need to return stats since boot, whilst
NetworkStats persists data only for a limited amount of time.

It is necessary to accumulate stats internally to return
correct data after time since boot exceeds persistence window.

Bug: 352537247
Test: atest FrameworksServicesTests:NetworkStatsAccumulatorTest
Flag: com.android.server.stats.accumulate_network_stats_since_boot
Change-Id: Icdd641dba8d359ae5e054287af042a23082a896f
parent 1f22de8f

services/core/java/com/android/server/stats/pull/StatsPullAtomService.java

+46 −7
Original line number Original line Diff line number Diff line
@@ -63,6 +63,7 @@ import static com.android.internal.util.FrameworkStatsLog.TIME_ZONE_DETECTOR_STA 
import static com.android.internal.util.FrameworkStatsLog.TIME_ZONE_DETECTOR_STATE__DETECTION_MODE__TELEPHONY;

import static com.android.internal.util.FrameworkStatsLog.TIME_ZONE_DETECTOR_STATE__DETECTION_MODE__TELEPHONY;

import static com.android.internal.util.FrameworkStatsLog.TIME_ZONE_DETECTOR_STATE__DETECTION_MODE__UNKNOWN;

import static com.android.internal.util.FrameworkStatsLog.TIME_ZONE_DETECTOR_STATE__DETECTION_MODE__UNKNOWN;

import static com.android.server.am.MemoryStatUtil.readMemoryStatFromFilesystem;

import static com.android.server.am.MemoryStatUtil.readMemoryStatFromFilesystem;

import static com.android.server.stats.Flags.accumulateNetworkStatsSinceBoot;

import static com.android.server.stats.Flags.addMobileBytesTransferByProcStatePuller;

import static com.android.server.stats.Flags.addMobileBytesTransferByProcStatePuller;

import static com.android.server.stats.Flags.applyNetworkStatsPollRateLimit;

import static com.android.server.stats.Flags.applyNetworkStatsPollRateLimit;

import static com.android.server.stats.pull.IonMemoryUtil.readProcessSystemIonHeapSizesFromDebugfs;

import static com.android.server.stats.pull.IonMemoryUtil.readProcessSystemIonHeapSizesFromDebugfs;
@@ -229,6 +230,7 @@ import com.android.server.power.stats.KernelWakelockStats; 
import com.android.server.power.stats.SystemServerCpuThreadReader.SystemServiceCpuThreadTimes;

import com.android.server.power.stats.SystemServerCpuThreadReader.SystemServiceCpuThreadTimes;

import com.android.server.stats.pull.IonMemoryUtil.IonAllocations;

import com.android.server.stats.pull.IonMemoryUtil.IonAllocations;

import com.android.server.stats.pull.ProcfsMemoryUtil.MemorySnapshot;

import com.android.server.stats.pull.ProcfsMemoryUtil.MemorySnapshot;

import com.android.server.stats.pull.netstats.NetworkStatsAccumulator;

import com.android.server.stats.pull.netstats.NetworkStatsExt;

import com.android.server.stats.pull.netstats.NetworkStatsExt;

import com.android.server.stats.pull.netstats.SubInfo;

import com.android.server.stats.pull.netstats.SubInfo;

import com.android.server.storage.DiskStatsFileLogger;

import com.android.server.storage.DiskStatsFileLogger;
@@ -424,6 +426,14 @@ public class StatsPullAtomService extends SystemService { 
    @GuardedBy("mDataBytesTransferLock")

    @GuardedBy("mDataBytesTransferLock")

    private final ArrayList<NetworkStatsExt> mNetworkStatsBaselines = new ArrayList<>();

    private final ArrayList<NetworkStatsExt> mNetworkStatsBaselines = new ArrayList<>();





    // Accumulates NetworkStats from initialization till the present moment.

    // It is necessary to accumulate stats internally, because NetworkStats persists data for a

    // limited amount of time, after which diff becomes incorrect without accumulation.

    @NonNull

    @GuardedBy("mDataBytesTransferLock")

    private final ArrayList<NetworkStatsAccumulator> mNetworkStatsAccumulators =

            new ArrayList<>();



    @GuardedBy("mDataBytesTransferLock")

    @GuardedBy("mDataBytesTransferLock")

    private long mLastNetworkStatsPollTime = -NETSTATS_POLL_RATE_LIMIT_MS;

    private long mLastNetworkStatsPollTime = -NETSTATS_POLL_RATE_LIMIT_MS;
@@ -1568,16 +1578,45 @@ public class StatsPullAtomService extends SystemService { 
    private NetworkStats getUidNetworkStatsSnapshotForTemplateLocked(

    private NetworkStats getUidNetworkStatsSnapshotForTemplateLocked(

            @NonNull NetworkTemplate template, boolean includeTags) {

            @NonNull NetworkTemplate template, boolean includeTags) {

        final long elapsedMillisSinceBoot = SystemClock.elapsedRealtime();

        final long elapsedMillisSinceBoot = SystemClock.elapsedRealtime();

        final long currentTimeInMillis = MICROSECONDS.toMillis(SystemClock.currentTimeMicro());

        final long currentTimeMillis = MICROSECONDS.toMillis(SystemClock.currentTimeMicro());

        final long bucketDuration = Settings.Global.getLong(mContext.getContentResolver(),

        final long bootTimeMillis = currentTimeMillis - elapsedMillisSinceBoot;

        final long bucketDurationMillis = Settings.Global.getLong(mContext.getContentResolver(),

                NETSTATS_UID_BUCKET_DURATION, NETSTATS_UID_DEFAULT_BUCKET_DURATION_MS);

                NETSTATS_UID_BUCKET_DURATION, NETSTATS_UID_DEFAULT_BUCKET_DURATION_MS);





        // Set startTime before boot so that NetworkStats includes at least one full bucket.

        if (accumulateNetworkStatsSinceBoot()) {

        // Set endTime in the future so that NetworkStats includes everything in the active bucket.

            NetworkStatsAccumulator accumulator = CollectionUtils.find(

        final long startTime = currentTimeInMillis - elapsedMillisSinceBoot - bucketDuration;

                    mNetworkStatsAccumulators, it -> it.hasEqualParameters(template, includeTags));

        final long endTime = currentTimeInMillis + bucketDuration;

            if (accumulator == null) {

                accumulator = new NetworkStatsAccumulator(

                        template,

                        includeTags,

                        bucketDurationMillis,

                        bootTimeMillis - bucketDurationMillis);

                mNetworkStatsAccumulators.add(accumulator);

            }



            return accumulator.queryStats(currentTimeMillis,

                    (aTemplate, aIncludeTags, aStartTime, aEndTime) -> {

                        synchronized (mDataBytesTransferLock) {

                            return getUidNetworkStatsSnapshotForTemplateLocked(aTemplate,

                                    aIncludeTags, aStartTime, aEndTime);

                        }

                    });



        } else {

            // Set end time in the future to include all stats in the active bucket.

            return getUidNetworkStatsSnapshotForTemplateLocked(

                    template, includeTags,

                    bootTimeMillis - bucketDurationMillis,

                    currentTimeMillis + bucketDurationMillis);

        }

    }





        // NetworkStatsManager#forceUpdate updates stats for all networks

    @GuardedBy("mDataBytesTransferLock")

    @Nullable

    private NetworkStats getUidNetworkStatsSnapshotForTemplateLocked(

            @NonNull NetworkTemplate template, boolean includeTags, long startTime, long endTime) {

        final long elapsedMillisSinceBoot = SystemClock.elapsedRealtime();

        if (applyNetworkStatsPollRateLimit()) {

        if (applyNetworkStatsPollRateLimit()) {

            // The new way: rate-limit force-polling for all NetworkStats queries

            // The new way: rate-limit force-polling for all NetworkStats queries

            if (elapsedMillisSinceBoot - mLastNetworkStatsPollTime >= NETSTATS_POLL_RATE_LIMIT_MS) {

            if (elapsedMillisSinceBoot - mLastNetworkStatsPollTime >= NETSTATS_POLL_RATE_LIMIT_MS) {

services/core/java/com/android/server/stats/pull/netstats/NetworkStatsAccumulator.java

0 → 100644
+111 −0
Original line number Original line Diff line number Diff line 
/*

 * Copyright (C) 2024 The Android Open Source Project

 *

 * Licensed under the Apache License, Version 2.0 (the "License");

 * you may not use this file except in compliance with the License.

 * You may obtain a copy of the License at

 *

 *      http://www.apache.org/licenses/LICENSE-2.0

 *

 * Unless required by applicable law or agreed to in writing, software

 * distributed under the License is distributed on an "AS IS" BASIS,

 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

 * See the License for the specific language governing permissions and

 * limitations under the License.

 */



package com.android.server.stats.pull.netstats;



import android.annotation.NonNull;

import android.annotation.Nullable;

import android.net.NetworkStats;

import android.net.NetworkTemplate;



import java.util.Objects;



/**

 * A class that queries NetworkStats, accumulates query results, and exposes cumulative stats across

 * the time range covered by the queries. This class is not thread-safe.

 * <p>

 * This is class should be used when querying NetworkStats since boot, as NetworkStats persists data

 * only for a limited period of time.

 *

 * @hide

 */

public class NetworkStatsAccumulator {



    private final NetworkTemplate mTemplate;

    private final boolean mWithTags;

    private final long mBucketDurationMillis;

    private NetworkStats mSnapshot;

    private long mSnapshotEndTimeMillis;



    public NetworkStatsAccumulator(@NonNull NetworkTemplate template, boolean withTags,

            long bucketDurationMillis, long snapshotEndTimeMillis) {

        mTemplate = template;

        mWithTags = withTags;

        mBucketDurationMillis = bucketDurationMillis;

        mSnapshot = new NetworkStats(0, 1);

        mSnapshotEndTimeMillis = snapshotEndTimeMillis;

    }



    /**

     * Provides cumulative NetworkStats until given timestamp.

     * <p>

     * This method method may call {@code queryFunction} more than once, which includes maintaining

     * an internal cumulative stats snapshot and querying stats after the snapshot.

     */

    @Nullable

    public NetworkStats queryStats(long currentTimeMillis,

            @NonNull StatsQueryFunction queryFunction) {

        maybeExpandSnapshot(currentTimeMillis, queryFunction);

        return snapshotPlusFollowingStats(currentTimeMillis, queryFunction);

    }



    /**

     * Returns true if the accumulator is using given query parameters.

     */

    public boolean hasEqualParameters(@NonNull NetworkTemplate template, boolean withTags) {

        return Objects.equals(mTemplate, template) && mWithTags == withTags;

    }



    /**

     * Expands the internal cumulative stats snapshot, if possible, by querying NetworkStats.

     */

    private void maybeExpandSnapshot(long currentTimeMillis,

            @NonNull StatsQueryFunction queryFunction) {

        // Update snapshot only if it is possible to expand it by at least one full bucket, and only

        // if the new snapshot's end is not in the active bucket.

        long newEndTimeMillis = currentTimeMillis - mBucketDurationMillis;

        if (newEndTimeMillis - mSnapshotEndTimeMillis > mBucketDurationMillis) {

            NetworkStats extraStats = queryFunction.queryNetworkStats(mTemplate, mWithTags,

                    mSnapshotEndTimeMillis, newEndTimeMillis);

            if (extraStats != null) {

                mSnapshot = mSnapshot.add(extraStats);

                mSnapshotEndTimeMillis = newEndTimeMillis;

            }

        }

    }



    /**

     * Adds up stats in the internal cumulative snapshot and the stats that follow after it.

     */

    @Nullable

    private NetworkStats snapshotPlusFollowingStats(long currentTimeMillis,

            @NonNull StatsQueryFunction queryFunction) {

        // Set end time in the future to include all stats in the active bucket.

        NetworkStats extraStats = queryFunction.queryNetworkStats(mTemplate, mWithTags,

                mSnapshotEndTimeMillis, currentTimeMillis + mBucketDurationMillis);

        return extraStats != null ? mSnapshot.add(extraStats) : null;

    }



    @FunctionalInterface

    public interface StatsQueryFunction {

        /**

         * Returns network stats during the given time period.

         */

        @Nullable

        NetworkStats queryNetworkStats(@NonNull NetworkTemplate template, boolean includeTags,

                long startTime, long endTime);

    }

}

services/tests/servicestests/src/com/android/server/stats/pull/netstats/NetworkStatsAccumulatorTest.kt

0 → 100644
+189 −0
Original line number Original line Diff line number Diff line 
/*

 * Copyright (C) 2024 The Android Open Source Project

 *

 * Licensed under the Apache License, Version 2.0 (the "License");

 * you may not use this file except in compliance with the License.

 * You may obtain a copy of the License at

 *

 *      http://www.apache.org/licenses/LICENSE-2.0

 *

 * Unless required by applicable law or agreed to in writing, software

 * distributed under the License is distributed on an "AS IS" BASIS,

 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

 * See the License for the specific language governing permissions and

 * limitations under the License.

 */



package com.android.server.stats.pull.netstats



import android.net.NetworkStats

import android.net.NetworkStats.DEFAULT_NETWORK_YES

import android.net.NetworkStats.METERED_NO

import android.net.NetworkStats.ROAMING_NO

import android.net.NetworkStats.SET_DEFAULT

import android.net.NetworkStats.TAG_NONE

import android.net.NetworkTemplate

import androidx.test.ext.junit.runners.AndroidJUnit4

import com.android.testutils.assertNetworkStatsEquals

import com.google.common.truth.Truth.assertThat

import org.junit.Test

import org.junit.runner.RunWith



@RunWith(AndroidJUnit4::class)

class NetworkStatsAccumulatorTest {



    @Test

    fun hasEqualParameters_differentParameters_returnsFalse() {

        val wifiTemplate = NetworkTemplate.Builder(NetworkTemplate.MATCH_WIFI).build()

        val mobileTemplate = NetworkTemplate.Builder(NetworkTemplate.MATCH_MOBILE).build()



        val snapshot = NetworkStatsAccumulator(wifiTemplate, false, 0, 0)



        assertThat(snapshot.hasEqualParameters(mobileTemplate, false)).isFalse()

        assertThat(snapshot.hasEqualParameters(wifiTemplate, true)).isFalse()

    }



    @Test

    fun hasSameParameters_equalParameters_returnsTrue() {

        val wifiTemplate1 = NetworkTemplate.Builder(NetworkTemplate.MATCH_WIFI).build()

        val wifiTemplate2 = NetworkTemplate.Builder(NetworkTemplate.MATCH_WIFI).build()



        val snapshot = NetworkStatsAccumulator(wifiTemplate1, false, 0, 0)



        assertThat(snapshot.hasEqualParameters(wifiTemplate1, false)).isTrue()

        assertThat(snapshot.hasEqualParameters(wifiTemplate2, false)).isTrue()

    }



    @Test

    fun queryStats_lessThanOneBucketFromSnapshotEndTime_returnsAllStats() {

        val snapshot = NetworkStatsAccumulator(TEMPLATE, false, 200, 1000)



        // Accumulator has data until 1000 (= 0), and its end-point is still in the history period.

        // Current time is less than one bucket away from snapshot end-point: 1050 - 1000 < 200

        val stats = snapshot.queryStats(1050, FakeStats(500, 1050, 1))!!



        // After the query at 1050, accumulator should have 1 * (1050 - 1000) = 50 bytes.

        assertNetworkStatsEquals(stats, networkStatsWithBytes(50))

    }



    @Test

    fun queryStats_oneBucketFromSnapshotEndTime_returnsCumulativeStats() {

        val snapshot = NetworkStatsAccumulator(TEMPLATE, false, 200, 1000)



        // Accumulator has data until 1000 (= 0), and its end-point is still in the history period.

        // Current time is one bucket away from snapshot end-point: 1250 - 1000 > 200

        val stats = snapshot.queryStats(1250, FakeStats(550, 1250, 2))!!



        // After the query at 1250, accumulator should have 2 * (1250 - 1000) = 500 bytes.

        assertNetworkStatsEquals(stats, networkStatsWithBytes(500))

    }



    @Test

    fun queryStats_twoBucketsFromSnapshotEndTime_returnsCumulativeStats() {

        val snapshot = NetworkStatsAccumulator(TEMPLATE, false, 200, 1000)



        // Accumulator has data until 1000 (= 0), and its end-point is in the history period.

        // Current time is two buckets away from snapshot end-point: 1450 - 1000 > 2*200

        val stats = snapshot.queryStats(1450, FakeStats(600, 1450, 3))!!



        // After the query at 1450, accumulator should have 3 * (1450 - 1000) = 1350 bytes.

        assertNetworkStatsEquals(stats, networkStatsWithBytes(1350))

    }



    @Test

    fun queryStats_manyBucketsFromSnapshotEndTime_returnsCumulativeStats() {

        val snapshot = NetworkStatsAccumulator(TEMPLATE, false, 200, 1000)



        // Accumulator has data until 1000 (= 0), and its end-point is still in the history period.

        // Current time is many buckets away from snapshot end-point

        val stats = snapshot.queryStats(6100, FakeStats(900, 6100, 1))!!



        // After the query at 6100, accumulator should have 1 * (6100 - 1000) = 5100 bytes.

        assertNetworkStatsEquals(stats, networkStatsWithBytes(5100))

    }



    @Test

    fun queryStats_multipleQueriesAndSameHistoryWindow_returnsCumulativeStats() {

        val snapshot = NetworkStatsAccumulator(TEMPLATE, false, 200, 1000)



        // Accumulator is queried within the history period, whose starting point stays the same.

        // After each query, accumulator should contain bytes from the initial end-point until now.

        val stats1 = snapshot.queryStats(5100, FakeStats(900, 5100, 1))!!

        val stats2 = snapshot.queryStats(10100, FakeStats(900, 10100, 1))!!

        val stats3 = snapshot.queryStats(15100, FakeStats(900, 15100, 1))!!



        assertNetworkStatsEquals(stats1, networkStatsWithBytes(4100))

        assertNetworkStatsEquals(stats2, networkStatsWithBytes(9100))

        assertNetworkStatsEquals(stats3, networkStatsWithBytes(14100))

    }



    @Test

    fun queryStats_multipleQueriesAndSlidingHistoryWindow_returnsCumulativeStats() {

        val snapshot = NetworkStatsAccumulator(TEMPLATE, false, 200, 1000)



        // Accumulator is queried within the history period, whose starting point is moving.

        // After each query, accumulator should contain bytes from the initial end-point until now.

        val stats1 = snapshot.queryStats(5100, FakeStats(900, 5100, 1))!!

        val stats2 = snapshot.queryStats(10100, FakeStats(4000, 10100, 1))!!

        val stats3 = snapshot.queryStats(15100, FakeStats(7000, 15100, 1))!!



        assertNetworkStatsEquals(stats1, networkStatsWithBytes(4100))

        assertNetworkStatsEquals(stats2, networkStatsWithBytes(9100))

        assertNetworkStatsEquals(stats3, networkStatsWithBytes(14100))

    }



    @Test

    fun queryStats_withSnapshotEndTimeBeforeHistoryStart_addsOnlyStatsWithinHistory() {

        val snapshot = NetworkStatsAccumulator(TEMPLATE, false, 200, 1900)



        // Accumulator has data until 1000 (= 0), but its end-point is not in the history period.

        // After the query, accumulator should add only those bytes that are covered by the history.

        val stats = snapshot.queryStats(2700, FakeStats(2200, 2700, 1))!!



        assertNetworkStatsEquals(stats, networkStatsWithBytes(500))

    }



    /**

     * Simulates equally distributed traffic stats persisted over a set period of time.

     */

    private class FakeStats(

        val historyStartMillis: Long, val currentTimeMillis: Long, val bytesPerMilli: Long

    ) : NetworkStatsAccumulator.StatsQueryFunction {



        override fun queryNetworkStats(

            template: NetworkTemplate, includeTags: Boolean, startTime: Long, endTime: Long

        ): NetworkStats {

            val overlap = overlap(startTime, endTime, historyStartMillis, currentTimeMillis)

            return networkStatsWithBytes(overlap * bytesPerMilli)

        }

    }



    companion object {



        private val TEMPLATE = NetworkTemplate.Builder(NetworkTemplate.MATCH_WIFI).build()



        fun networkStatsWithBytes(bytes: Long): NetworkStats {

            val stats = NetworkStats(0, 1).addEntry(

                NetworkStats.Entry(

                    null,

                    0,

                    SET_DEFAULT,

                    TAG_NONE,

                    METERED_NO,

                    ROAMING_NO,

                    DEFAULT_NETWORK_YES,

                    bytes,

                    bytes / 100,

                    bytes,

                    bytes / 100,

                    0

                )

            )

            return stats

        }



        fun overlap(aStart: Long, aEnd: Long, bStart: Long, bEnd: Long): Long {

            return maxOf(0L, minOf(aEnd, bEnd) - maxOf(aStart, bStart))

        }

    }

}
 
 No newline at end of file