Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 61ee0bbb authored by Jeff Sharkey's avatar Jeff Sharkey
Browse files

UID network stats, secure settings, and random. 

Collect UID-granularity network stats during regular poll event.  Add
dumpsys argument to generate fake historical data for debugging, and
move stats parameters to Settings.Secure.

Change-Id: I09b36a2955dc10c697d4b9c3ff23dcb3ac37bd70
parent d2a45875

core/java/android/net/NetworkStats.java

+19 −1
Original line number Diff line number Diff line
@@ -19,6 +19,7 @@ package android.net; 
import android.os.Parcel;

import android.os.Parcelable;

import android.os.SystemClock;

import android.util.SparseBooleanArray;



import java.io.CharArrayWriter;

import java.io.PrintWriter;
@@ -125,7 +126,7 @@ public class NetworkStats implements Parcelable { 
    /**

     * Return list of unique interfaces known by this data structure.

     */

    public String[] getKnownIfaces() {

    public String[] getUniqueIfaces() {

        final HashSet<String> ifaces = new HashSet<String>();

        for (String iface : this.iface) {

            if (iface != IFACE_ALL) {
@@ -135,6 +136,23 @@ public class NetworkStats implements Parcelable { 
        return ifaces.toArray(new String[ifaces.size()]);

    }



    /**

     * Return list of unique UIDs known by this data structure.

     */

    public int[] getUniqueUids() {

        final SparseBooleanArray uids = new SparseBooleanArray();

        for (int uid : this.uid) {

            uids.put(uid, true);

        }



        final int size = uids.size();

        final int[] result = new int[size];

        for (int i = 0; i < size; i++) {

            result[i] = uids.keyAt(i);

        }

        return result;

    }



    /**

     * Subtract the given {@link NetworkStats}, effectively leaving the delta

     * between two snapshots in time. Assumes that statistics rows collect over

core/java/android/net/NetworkStatsHistory.java

+44 −9
Original line number Diff line number Diff line
@@ -24,7 +24,9 @@ import java.io.DataInputStream; 
import java.io.DataOutputStream;

import java.io.IOException;

import java.io.PrintWriter;

import java.net.ProtocolException;

import java.util.Arrays;

import java.util.Random;



/**

 * Collection of historical network statistics, recorded into equally-sized
@@ -38,7 +40,7 @@ import java.util.Arrays; 
 * @hide

 */

public class NetworkStatsHistory implements Parcelable {

    private static final int VERSION = 1;

    private static final int VERSION_CURRENT = 1;



    // TODO: teach about zigzag encoding to use less disk space

    // TODO: teach how to convert between bucket sizes
@@ -76,15 +78,23 @@ public class NetworkStatsHistory implements Parcelable { 


    public NetworkStatsHistory(DataInputStream in) throws IOException {

        final int version = in.readInt();

        switch (version) {

            case VERSION_CURRENT: {

                bucketDuration = in.readLong();

                bucketStart = readLongArray(in);

                rx = readLongArray(in);

                tx = readLongArray(in);

                bucketCount = bucketStart.length;

                break;

            }

            default: {

                throw new ProtocolException("unexpected version: " + version);

            }

        }

    }



    public void writeToStream(DataOutputStream out) throws IOException {

        out.writeInt(VERSION);

        out.writeInt(VERSION_CURRENT);

        out.writeLong(bucketDuration);

        writeLongArray(out, bucketStart, bucketCount);

        writeLongArray(out, rx, bucketCount);
@@ -192,12 +202,37 @@ public class NetworkStatsHistory implements Parcelable { 
        }

    }



    /**

     * @deprecated only for temporary testing

     */

    @Deprecated

    public void generateRandom(long start, long end, long rx, long tx) {

        ensureBuckets(start, end);



        final Random r = new Random();

        while (rx > 1024 && tx > 1024) {

            final long curStart = randomLong(r, start, end);

            final long curEnd = randomLong(r, curStart, end);

            final long curRx = randomLong(r, 0, rx);

            final long curTx = randomLong(r, 0, tx);



            recordData(curStart, curEnd, curRx, curTx);



            rx -= curRx;

            tx -= curTx;

        }

    }



    private static long randomLong(Random r, long start, long end) {

        return (long) (start + (r.nextFloat() * (end - start)));

    }



    public void dump(String prefix, PrintWriter pw) {

        pw.print(prefix);

        pw.println("NetworkStatsHistory:");

        pw.print("NetworkStatsHistory: bucketDuration="); pw.println(bucketDuration);

        for (int i = 0; i < bucketCount; i++) {

            pw.print(prefix);

            pw.print("  timestamp="); pw.print(bucketStart[i]);

            pw.print("  bucketStart="); pw.print(bucketStart[i]);

            pw.print(" rx="); pw.print(rx[i]);

            pw.print(" tx="); pw.println(tx[i]);

        }

core/java/android/provider/Settings.java

+13 −0
Original line number Diff line number Diff line
@@ -3795,6 +3795,19 @@ public final class Settings { 
        public static final String DREAM_TIMEOUT =

                "dream_timeout";



        /** {@hide} */

        public static final String NETSTATS_POLL_INTERVAL = "netstats_poll_interval";

        /** {@hide} */

        public static final String NETSTATS_PERSIST_THRESHOLD = "netstats_persist_threshold";

        /** {@hide} */

        public static final String NETSTATS_SUMMARY_BUCKET_DURATION = "netstats_summary_bucket_duration";

        /** {@hide} */

        public static final String NETSTATS_SUMMARY_MAX_HISTORY = "netstats_summary_max_history";

        /** {@hide} */

        public static final String NETSTATS_DETAIL_BUCKET_DURATION = "netstats_detail_bucket_duration";

        /** {@hide} */

        public static final String NETSTATS_DETAIL_MAX_HISTORY = "netstats_detail_max_history";



        /**

         * @hide

         */

services/java/com/android/server/net/NetworkStatsService.java

+188 −57
Original line number Diff line number Diff line
@@ -21,7 +21,17 @@ import static android.Manifest.permission.DUMP; 
import static android.Manifest.permission.SHUTDOWN;

import static android.Manifest.permission.UPDATE_DEVICE_STATS;

import static android.net.ConnectivityManager.CONNECTIVITY_ACTION;

import static android.net.NetworkStats.IFACE_ALL;

import static android.net.NetworkStats.UID_ALL;

import static android.provider.Settings.Secure.NETSTATS_DETAIL_BUCKET_DURATION;

import static android.provider.Settings.Secure.NETSTATS_DETAIL_MAX_HISTORY;

import static android.provider.Settings.Secure.NETSTATS_PERSIST_THRESHOLD;

import static android.provider.Settings.Secure.NETSTATS_POLL_INTERVAL;

import static android.provider.Settings.Secure.NETSTATS_SUMMARY_BUCKET_DURATION;

import static android.provider.Settings.Secure.NETSTATS_SUMMARY_MAX_HISTORY;

import static android.text.format.DateUtils.DAY_IN_MILLIS;

import static android.text.format.DateUtils.HOUR_IN_MILLIS;

import static android.text.format.DateUtils.MINUTE_IN_MILLIS;

import static com.android.internal.util.Preconditions.checkNotNull;



import android.app.AlarmManager;
@@ -31,6 +41,7 @@ import android.content.BroadcastReceiver; 
import android.content.Context;

import android.content.Intent;

import android.content.IntentFilter;

import android.content.pm.ApplicationInfo;

import android.net.IConnectivityManager;

import android.net.INetworkStatsService;

import android.net.NetworkInfo;
@@ -42,10 +53,11 @@ import android.os.HandlerThread; 
import android.os.INetworkManagementService;

import android.os.RemoteException;

import android.os.SystemClock;

import android.provider.Settings;

import android.telephony.TelephonyManager;

import android.text.format.DateUtils;

import android.util.NtpTrustedTime;

import android.util.Slog;

import android.util.SparseArray;

import android.util.TrustedTime;



import com.google.android.collect.Lists;
@@ -55,6 +67,7 @@ import java.io.FileDescriptor; 
import java.io.PrintWriter;

import java.util.ArrayList;

import java.util.HashMap;

import java.util.HashSet;



/**

 * Collect and persist detailed network statistics, and provide this data to
@@ -76,34 +89,42 @@ public class NetworkStatsService extends INetworkStatsService.Stub { 


    private PendingIntent mPollIntent;



    // TODO: move tweakable params to Settings.Secure

    // TODO: listen for kernel push events through netd instead of polling



    private static final long KB_IN_BYTES = 1024;

    private static final long MB_IN_BYTES = 1024 * KB_IN_BYTES;

    private static final long GB_IN_BYTES = 1024 * MB_IN_BYTES;



    private static final long POLL_INTERVAL = AlarmManager.INTERVAL_FIFTEEN_MINUTES;

    private static final long SUMMARY_BUCKET_DURATION = 6 * DateUtils.HOUR_IN_MILLIS;

    private static final long SUMMARY_MAX_HISTORY = 90 * DateUtils.DAY_IN_MILLIS;

    private LongSecureSetting mPollInterval = new LongSecureSetting(

            NETSTATS_POLL_INTERVAL, 15 * MINUTE_IN_MILLIS);

    private LongSecureSetting mPersistThreshold = new LongSecureSetting(

            NETSTATS_PERSIST_THRESHOLD, 64 * KB_IN_BYTES);



    // TODO: remove these high-frequency testing values

//    private static final long POLL_INTERVAL = 5 * DateUtils.SECOND_IN_MILLIS;

//    private static final long SUMMARY_BUCKET_DURATION = 10 * DateUtils.SECOND_IN_MILLIS;

//    private static final long SUMMARY_MAX_HISTORY = 2 * DateUtils.MINUTE_IN_MILLIS;

    private LongSecureSetting mSummaryBucketDuration = new LongSecureSetting(

            NETSTATS_SUMMARY_BUCKET_DURATION, 6 * HOUR_IN_MILLIS);

    private LongSecureSetting mSummaryMaxHistory = new LongSecureSetting(

            NETSTATS_SUMMARY_MAX_HISTORY, 90 * DAY_IN_MILLIS);

    private LongSecureSetting mDetailBucketDuration = new LongSecureSetting(

            NETSTATS_DETAIL_BUCKET_DURATION, 6 * HOUR_IN_MILLIS);

    private LongSecureSetting mDetailMaxHistory = new LongSecureSetting(

            NETSTATS_DETAIL_MAX_HISTORY, 90 * DAY_IN_MILLIS);



    /** Minimum delta required to persist to disk. */

    private static final long SUMMARY_PERSIST_THRESHOLD = 64 * KB_IN_BYTES;



    private static final long TIME_CACHE_MAX_AGE = DateUtils.DAY_IN_MILLIS;

    private static final long TIME_CACHE_MAX_AGE = DAY_IN_MILLIS;



    private final Object mStatsLock = new Object();



    /** Set of active ifaces during this boot. */

    private HashMap<String, InterfaceIdentity> mActiveIface = Maps.newHashMap();



    /** Set of historical stats for known ifaces. */

    private HashMap<InterfaceIdentity, NetworkStatsHistory> mStats = Maps.newHashMap();

    private HashMap<InterfaceIdentity, NetworkStatsHistory> mSummaryStats = Maps.newHashMap();

    /** Set of historical stats for known UIDs. */

    private SparseArray<NetworkStatsHistory> mDetailStats = new SparseArray<NetworkStatsHistory>();



    private NetworkStats mLastPollStats;

    private NetworkStats mLastPersistStats;

    private NetworkStats mLastSummaryPoll;

    private NetworkStats mLastSummaryPersist;



    private NetworkStats mLastDetailPoll;



    private final HandlerThread mHandlerThread;

    private final Handler mHandler;
@@ -161,7 +182,7 @@ public class NetworkStatsService extends INetworkStatsService.Stub { 


    /**

     * Clear any existing {@link #ACTION_NETWORK_STATS_POLL} alarms, and

     * reschedule based on current {@link #POLL_INTERVAL} value.

     * reschedule based on current {@link #mPollInterval} value.

     */

    private void registerPollAlarmLocked() throws RemoteException {

        if (mPollIntent != null) {
@@ -173,7 +194,7 @@ public class NetworkStatsService extends INetworkStatsService.Stub { 


        final long currentRealtime = SystemClock.elapsedRealtime();

        mAlarmManager.setInexactRepeating(

                AlarmManager.ELAPSED_REALTIME, currentRealtime, POLL_INTERVAL, mPollIntent);

                AlarmManager.ELAPSED_REALTIME, currentRealtime, mPollInterval.get(), mPollIntent);

    }



    @Override
@@ -184,9 +205,10 @@ public class NetworkStatsService extends INetworkStatsService.Stub { 
        synchronized (mStatsLock) {

            // combine all interfaces that match template

            final String subscriberId = getActiveSubscriberId();

            final NetworkStatsHistory combined = new NetworkStatsHistory(SUMMARY_BUCKET_DURATION);

            for (InterfaceIdentity ident : mStats.keySet()) {

                final NetworkStatsHistory history = mStats.get(ident);

            final NetworkStatsHistory combined = new NetworkStatsHistory(

                    mSummaryBucketDuration.get());

            for (InterfaceIdentity ident : mSummaryStats.keySet()) {

                final NetworkStatsHistory history = mSummaryStats.get(ident);

                if (ident.matchesTemplate(networkTemplate, subscriberId)) {

                    // TODO: combine all matching history data into a single history

                }
@@ -299,59 +321,97 @@ public class NetworkStatsService extends INetworkStatsService.Stub { 
        final long currentTime = mTime.hasCache() ? mTime.currentTimeMillis()

                : System.currentTimeMillis();



        final NetworkStats current;

        final NetworkStats summary;

        final NetworkStats detail;

        try {

            current = mNetworkManager.getNetworkStatsSummary();

            summary = mNetworkManager.getNetworkStatsSummary();

            detail = mNetworkManager.getNetworkStatsDetail();

        } catch (RemoteException e) {

            Slog.w(TAG, "problem reading network stats");

            return;

        }



        performSummaryPollLocked(summary, currentTime);

        performDetailPollLocked(detail, currentTime);



        // decide if enough has changed to trigger persist

        final NetworkStats persistDelta = computeStatsDelta(mLastSummaryPersist, summary);

        final long persistThreshold = mPersistThreshold.get();

        for (String iface : persistDelta.getUniqueIfaces()) {

            final int index = persistDelta.findIndex(iface, UID_ALL);

            if (persistDelta.rx[index] > persistThreshold

                    || persistDelta.tx[index] > persistThreshold) {

                writeStatsLocked();

                mLastSummaryPersist = summary;

                break;

            }

        }

    }



    /**

     * Update {@link #mSummaryStats} historical usage.

     */

    private void performSummaryPollLocked(NetworkStats summary, long currentTime) {

        final ArrayList<String> unknownIface = Lists.newArrayList();



        // update historical usage with delta since last poll

        final NetworkStats pollDelta = computeStatsDelta(mLastPollStats, current);

        final long timeStart = currentTime - pollDelta.elapsedRealtime;

        for (String iface : pollDelta.getKnownIfaces()) {

        final NetworkStats delta = computeStatsDelta(mLastSummaryPoll, summary);

        final long timeStart = currentTime - delta.elapsedRealtime;

        final long maxHistory = mSummaryMaxHistory.get();

        for (String iface : delta.getUniqueIfaces()) {

            final InterfaceIdentity ident = mActiveIface.get(iface);

            if (ident == null) {

                unknownIface.add(iface);

                continue;

            }



            final int index = pollDelta.findIndex(iface, UID_ALL);

            final long rx = pollDelta.rx[index];

            final long tx = pollDelta.tx[index];

            final int index = delta.findIndex(iface, UID_ALL);

            final long rx = delta.rx[index];

            final long tx = delta.tx[index];



            final NetworkStatsHistory history = findOrCreateHistoryLocked(ident);

            final NetworkStatsHistory history = findOrCreateSummaryLocked(ident);

            history.recordData(timeStart, currentTime, rx, tx);

            history.removeBucketsBefore(currentTime - SUMMARY_MAX_HISTORY);

            history.removeBucketsBefore(currentTime - maxHistory);

        }

        mLastSummaryPoll = summary;



        if (LOGD && unknownIface.size() > 0) {

            Slog.w(TAG, "unknown interfaces " + unknownIface.toString() + ", ignoring those stats");

        }

    }



        mLastPollStats = current;



        // decide if enough has changed to trigger persist

        final NetworkStats persistDelta = computeStatsDelta(mLastPersistStats, current);

        for (String iface : persistDelta.getKnownIfaces()) {

            final int index = persistDelta.findIndex(iface, UID_ALL);

            if (persistDelta.rx[index] > SUMMARY_PERSIST_THRESHOLD

                    || persistDelta.tx[index] > SUMMARY_PERSIST_THRESHOLD) {

                writeStatsLocked();

                mLastPersistStats = current;

                break;

    /**

     * Update {@link #mDetailStats} historical usage.

     */

    private void performDetailPollLocked(NetworkStats detail, long currentTime) {

        final NetworkStats delta = computeStatsDelta(mLastDetailPoll, detail);

        final long timeStart = currentTime - delta.elapsedRealtime;

        final long maxHistory = mDetailMaxHistory.get();

        for (int uid : delta.getUniqueUids()) {

            final int index = delta.findIndex(IFACE_ALL, uid);

            final long rx = delta.rx[index];

            final long tx = delta.tx[index];



            final NetworkStatsHistory history = findOrCreateDetailLocked(uid);

            history.recordData(timeStart, currentTime, rx, tx);

            history.removeBucketsBefore(currentTime - maxHistory);

        }

        mLastDetailPoll = detail;

    }



    private NetworkStatsHistory findOrCreateSummaryLocked(InterfaceIdentity ident) {

        NetworkStatsHistory stats = mSummaryStats.get(ident);

        if (stats == null) {

            stats = new NetworkStatsHistory(mSummaryBucketDuration.get());

            mSummaryStats.put(ident, stats);

        }

        return stats;

    }



    private NetworkStatsHistory findOrCreateHistoryLocked(InterfaceIdentity ident) {

        NetworkStatsHistory stats = mStats.get(ident);

    private NetworkStatsHistory findOrCreateDetailLocked(int uid) {

        NetworkStatsHistory stats = mDetailStats.get(uid);

        if (stats == null) {

            stats = new NetworkStatsHistory(SUMMARY_BUCKET_DURATION);

            mStats.put(ident, stats);

            stats = new NetworkStatsHistory(mDetailBucketDuration.get());

            mDetailStats.put(uid, stats);

        }

        return stats;

    }
@@ -380,6 +440,19 @@ public class NetworkStatsService extends INetworkStatsService.Stub { 
    protected void dump(FileDescriptor fd, PrintWriter pw, String[] args) {

        mContext.enforceCallingOrSelfPermission(DUMP, TAG);



        final HashSet<String> argSet = new HashSet<String>();

        for (String arg : args) {

            argSet.add(arg);

        }



        synchronized (mStatsLock) {

            // TODO: remove this testing code, since it corrupts stats

            if (argSet.contains("generate")) {

                generateRandomLocked();

                pw.println("Generated stub stats");

                return;

            }



            pw.println("Active interfaces:");

            for (String iface : mActiveIface.keySet()) {

                final InterfaceIdentity ident = mActiveIface.get(iface);
@@ -388,11 +461,69 @@ public class NetworkStatsService extends INetworkStatsService.Stub { 
            }



            pw.println("Known historical stats:");

        for (InterfaceIdentity ident : mStats.keySet()) {

            final NetworkStatsHistory stats = mStats.get(ident);

            for (InterfaceIdentity ident : mSummaryStats.keySet()) {

                final NetworkStatsHistory stats = mSummaryStats.get(ident);

                pw.print("  ident="); pw.println(ident.toString());

                stats.dump("    ", pw);

            }



            if (argSet.contains("detail")) {

                pw.println("Known detail stats:");

                for (int i = 0; i < mDetailStats.size(); i++) {

                    final int uid = mDetailStats.keyAt(i);

                    final NetworkStatsHistory stats = mDetailStats.valueAt(i);

                    pw.print("  UID="); pw.println(uid);

                    stats.dump("    ", pw);

                }

            }

        }

    }



    /**

     * @deprecated only for temporary testing

     */

    @Deprecated

    private void generateRandomLocked() {

        long end = System.currentTimeMillis();

        long start = end - mSummaryMaxHistory.get();

        long rx = 3 * GB_IN_BYTES;

        long tx = 2 * GB_IN_BYTES;



        mSummaryStats.clear();

        for (InterfaceIdentity ident : mActiveIface.values()) {

            final NetworkStatsHistory stats = findOrCreateSummaryLocked(ident);

            stats.generateRandom(start, end, rx, tx);

        }



        end = System.currentTimeMillis();

        start = end - mDetailMaxHistory.get();

        rx = 500 * MB_IN_BYTES;

        tx = 100 * MB_IN_BYTES;



        mDetailStats.clear();

        for (ApplicationInfo info : mContext.getPackageManager().getInstalledApplications(0)) {

            final int uid = info.uid;

            final NetworkStatsHistory stats = findOrCreateDetailLocked(uid);

            stats.generateRandom(start, end, rx, tx);

        }

    }



    private class LongSecureSetting {

        private String mKey;

        private long mDefaultValue;



        public LongSecureSetting(String key, long defaultValue) {

            mKey = key;

            mDefaultValue = defaultValue;

        }



        public long get() {

            if (mContext != null) {

                return Settings.Secure.getLong(mContext.getContentResolver(), mKey, mDefaultValue);

            } else {

                return mDefaultValue;

            }

        }

    }



    /**