Loading core/java/android/net/metrics/WakeupEvent.java 0 → 100644 +34 −0 Original line number Diff line number Diff line /* * Copyright (C) 2017 The Android Open Source Project * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package android.net.metrics; /** * An event logged when NFLOG notifies userspace of a wakeup packet for * watched interfaces. * {@hide} */ public class WakeupEvent { public String iface; public long timestampMs; public int uid; @Override public String toString() { return String.format("WakeupEvent(%tT.%tL, %s, uid: %d)", timestampMs, timestampMs, iface, uid); } } core/java/android/net/metrics/WakeupStats.java 0 → 100644 +87 −0 Original line number Diff line number Diff line /* * Copyright (C) 2017 The Android Open Source Project * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package android.net.metrics; import android.os.Process; import android.os.SystemClock; /** * An event logged per interface and that aggregates WakeupEvents for that interface. * {@hide} */ public class WakeupStats { private static final int NO_UID = -1; public final long creationTimeMs = SystemClock.elapsedRealtime(); public final String iface; public long totalWakeups = 0; public long rootWakeups = 0; public long systemWakeups = 0; public long nonApplicationWakeups = 0; public long applicationWakeups = 0; public long unroutedWakeups = 0; public long durationSec = 0; public WakeupStats(String iface) { this.iface = iface; } /** Update durationSec with current time. */ public void updateDuration() { durationSec = (SystemClock.elapsedRealtime() - creationTimeMs) / 1000; } /** Update wakeup counters for the given WakeupEvent. */ public void countEvent(WakeupEvent ev) { totalWakeups++; switch (ev.uid) { case Process.ROOT_UID: rootWakeups++; break; case Process.SYSTEM_UID: systemWakeups++; break; case NO_UID: unroutedWakeups++; break; default: if (ev.uid >= Process.FIRST_APPLICATION_UID) { applicationWakeups++; } else { nonApplicationWakeups++; } break; } } @Override public String toString() { updateDuration(); return new StringBuilder() .append("WakeupStats(").append(iface) .append(", total: ").append(totalWakeups) .append(", root: ").append(rootWakeups) .append(", system: ").append(systemWakeups) .append(", apps: ").append(applicationWakeups) .append(", non-apps: ").append(nonApplicationWakeups) .append(", unrouted: ").append(unroutedWakeups) .append(", ").append(durationSec).append("s)") .toString(); } } proto/src/ipconnectivity.proto +35 −0 Original line number Diff line number Diff line Loading @@ -473,6 +473,38 @@ message NetworkStats { repeated Pair validation_states = 8; } // Represents statistics from NFLOG wakeup events due to ingress packets. // Since oc-mr1. // Next tag: 8. message WakeupStats { // The time duration in seconds covered by these stats, for deriving // exact wakeup rates. optional int64 duration_sec = 1; // The total number of ingress packets waking up the device. optional int64 total_wakeups = 2; // The total number of wakeup packets routed to a socket belonging to // the root uid (uid 0). optional int64 root_wakeups = 3; // The total number of wakeup packets routed to a socket belonging to // the system server (uid 1000). optional int64 system_wakeups = 4; // The total number of wakeup packets routed to a socket belonging to // an application (uid > 9999). optional int64 application_wakeups = 5; // The total number of wakeup packets routed to a socket belonging to another // uid than the root uid, system uid, or an application uid (any uid in // between [1001, 9999]. See android.os.Process for possible uids. optional int64 non_application_wakeups = 6; // The total number of wakeup packets with no associated sockets. optional int64 unrouted_wakeups = 7; } // Represents one of the IP connectivity event defined in this file. // Next tag: 20 message IpConnectivityEvent { Loading Loading @@ -547,6 +579,9 @@ message IpConnectivityEvent { // Network statistics. NetworkStats network_stats = 19; // Ingress packet wakeup statistics. WakeupStats wakeup_stats = 20; }; }; Loading services/core/java/com/android/server/connectivity/IpConnectivityEventBuilder.java +17 −0 Original line number Diff line number Diff line Loading @@ -38,6 +38,7 @@ import android.net.metrics.IpReachabilityEvent; import android.net.metrics.NetworkEvent; import android.net.metrics.RaEvent; import android.net.metrics.ValidationProbeEvent; import android.net.metrics.WakeupStats; import android.os.Parcelable; import android.util.SparseArray; import android.util.SparseIntArray; Loading Loading @@ -115,6 +116,22 @@ final public class IpConnectivityEventBuilder { return out; } public static IpConnectivityEvent toProto(WakeupStats in) { IpConnectivityLogClass.WakeupStats wakeupStats = new IpConnectivityLogClass.WakeupStats(); in.updateDuration(); wakeupStats.durationSec = in.durationSec; wakeupStats.totalWakeups = in.totalWakeups; wakeupStats.rootWakeups = in.rootWakeups; wakeupStats.systemWakeups = in.systemWakeups; wakeupStats.nonApplicationWakeups = in.nonApplicationWakeups; wakeupStats.applicationWakeups = in.applicationWakeups; wakeupStats.unroutedWakeups = in.unroutedWakeups; final IpConnectivityEvent out = buildEvent(0, 0, in.iface); out.setWakeupStats(wakeupStats); return out; } private static IpConnectivityEvent buildEvent(int netId, long transports, String ifname) { final IpConnectivityEvent ev = new IpConnectivityEvent(); ev.networkId = netId; Loading services/core/java/com/android/server/connectivity/NetdEventListenerService.java +91 −7 Original line number Diff line number Diff line Loading @@ -16,6 +16,8 @@ package com.android.server.connectivity; import static android.util.TimeUtils.NANOS_PER_MS; import android.content.Context; import android.net.ConnectivityManager; import android.net.INetdEventCallback; Loading @@ -25,9 +27,12 @@ import android.net.metrics.ConnectStats; import android.net.metrics.DnsEvent; import android.net.metrics.INetdEventListener; import android.net.metrics.IpConnectivityLog; import android.net.metrics.WakeupEvent; import android.net.metrics.WakeupStats; import android.os.RemoteException; import android.text.format.DateUtils; import android.util.Log; import android.util.ArrayMap; import android.util.SparseArray; import com.android.internal.annotations.GuardedBy; import com.android.internal.annotations.VisibleForTesting; Loading Loading @@ -59,12 +64,28 @@ public class NetdEventListenerService extends INetdEventListener.Stub { private static final int CONNECT_LATENCY_FILL_RATE = 15 * (int) DateUtils.SECOND_IN_MILLIS; private static final int CONNECT_LATENCY_MAXIMUM_RECORDS = 20000; @VisibleForTesting static final int WAKEUP_EVENT_BUFFER_LENGTH = 1024; // TODO: dedup this String constant with the one used in // ConnectivityService#wakeupModifyInterface(). @VisibleForTesting static final String WAKEUP_EVENT_IFACE_PREFIX = "iface:"; // Sparse arrays of DNS and connect events, grouped by net id. @GuardedBy("this") private final SparseArray<DnsEvent> mDnsEvents = new SparseArray<>(); @GuardedBy("this") private final SparseArray<ConnectStats> mConnectEvents = new SparseArray<>(); // Array of aggregated wakeup event stats, grouped by interface name. @GuardedBy("this") private final ArrayMap<String, WakeupStats> mWakeupStats = new ArrayMap<>(); // Ring buffer array for storing packet wake up events sent by Netd. @GuardedBy("this") private final WakeupEvent[] mWakeupEvents = new WakeupEvent[WAKEUP_EVENT_BUFFER_LENGTH]; @GuardedBy("this") private long mWakeupEventCursor = 0; private final ConnectivityManager mCm; @GuardedBy("this") Loading Loading @@ -137,11 +158,62 @@ public class NetdEventListenerService extends INetdEventListener.Stub { @Override public synchronized void onWakeupEvent(String prefix, int uid, int gid, long timestampNs) { maybeVerboseLog("onWakeupEvent(%s, %d, %d, %sns)", prefix, uid, gid, timestampNs); // TODO: add ip protocol and port String iface = prefix.replaceFirst(WAKEUP_EVENT_IFACE_PREFIX, ""); final long timestampMs; if (timestampNs > 0) { timestampMs = timestampNs / NANOS_PER_MS; } else { timestampMs = System.currentTimeMillis(); } addWakupEvent(iface, timestampMs, uid); } @GuardedBy("this") private void addWakupEvent(String iface, long timestampMs, int uid) { int index = wakeupEventIndex(mWakeupEventCursor); mWakeupEventCursor++; WakeupEvent event = new WakeupEvent(); event.iface = iface; event.timestampMs = timestampMs; event.uid = uid; mWakeupEvents[index] = event; WakeupStats stats = mWakeupStats.get(iface); if (stats == null) { stats = new WakeupStats(iface); mWakeupStats.put(iface, stats); } stats.countEvent(event); } @GuardedBy("this") private WakeupEvent[] getWakeupEvents() { int length = (int) Math.min(mWakeupEventCursor, (long) mWakeupEvents.length); WakeupEvent[] out = new WakeupEvent[length]; // Reverse iteration from youngest event to oldest event. long inCursor = mWakeupEventCursor - 1; int outIdx = out.length - 1; while (outIdx >= 0) { out[outIdx--] = mWakeupEvents[wakeupEventIndex(inCursor--)]; } return out; } private static int wakeupEventIndex(long cursor) { return (int) Math.abs(cursor % WAKEUP_EVENT_BUFFER_LENGTH); } public synchronized void flushStatistics(List<IpConnectivityEvent> events) { flushProtos(events, mConnectEvents, IpConnectivityEventBuilder::toProto); flushProtos(events, mDnsEvents, IpConnectivityEventBuilder::toProto); for (int i = 0; i < mWakeupStats.size(); i++) { events.add(IpConnectivityEventBuilder.toProto(mWakeupStats.valueAt(i))); } mWakeupStats.clear(); } public synchronized void dump(PrintWriter writer) { Loading @@ -153,13 +225,22 @@ public class NetdEventListenerService extends INetdEventListener.Stub { } public synchronized void list(PrintWriter pw) { listEvents(pw, mConnectEvents, (x) -> x); listEvents(pw, mDnsEvents, (x) -> x); listEvents(pw, mConnectEvents, (x) -> x, "\n"); listEvents(pw, mDnsEvents, (x) -> x, "\n"); for (int i = 0; i < mWakeupStats.size(); i++) { pw.println(mWakeupStats.valueAt(i)); } for (WakeupEvent wakeup : getWakeupEvents()) { pw.println(wakeup); } } public synchronized void listAsProtos(PrintWriter pw) { listEvents(pw, mConnectEvents, IpConnectivityEventBuilder::toProto); listEvents(pw, mDnsEvents, IpConnectivityEventBuilder::toProto); listEvents(pw, mConnectEvents, IpConnectivityEventBuilder::toProto, ""); listEvents(pw, mDnsEvents, IpConnectivityEventBuilder::toProto, ""); for (int i = 0; i < mWakeupStats.size(); i++) { pw.print(IpConnectivityEventBuilder.toProto(mWakeupStats.valueAt(i))); } } private static <T> void flushProtos(List<IpConnectivityEvent> out, SparseArray<T> in, Loading @@ -170,10 +251,13 @@ public class NetdEventListenerService extends INetdEventListener.Stub { in.clear(); } public static <T> void listEvents( PrintWriter pw, SparseArray<T> events, Function<T, Object> mapper) { private static <T> void listEvents( PrintWriter pw, SparseArray<T> events, Function<T, Object> mapper, String separator) { // Proto derived Classes have toString method that adds a \n at the end. // Let the caller control that by passing in the line separator explicitly. for (int i = 0; i < events.size(); i++) { pw.println(mapper.apply(events.valueAt(i)).toString()); pw.print(mapper.apply(events.valueAt(i))); pw.print(separator); } } Loading Loading
core/java/android/net/metrics/WakeupEvent.java 0 → 100644 +34 −0 Original line number Diff line number Diff line /* * Copyright (C) 2017 The Android Open Source Project * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package android.net.metrics; /** * An event logged when NFLOG notifies userspace of a wakeup packet for * watched interfaces. * {@hide} */ public class WakeupEvent { public String iface; public long timestampMs; public int uid; @Override public String toString() { return String.format("WakeupEvent(%tT.%tL, %s, uid: %d)", timestampMs, timestampMs, iface, uid); } }
core/java/android/net/metrics/WakeupStats.java 0 → 100644 +87 −0 Original line number Diff line number Diff line /* * Copyright (C) 2017 The Android Open Source Project * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package android.net.metrics; import android.os.Process; import android.os.SystemClock; /** * An event logged per interface and that aggregates WakeupEvents for that interface. * {@hide} */ public class WakeupStats { private static final int NO_UID = -1; public final long creationTimeMs = SystemClock.elapsedRealtime(); public final String iface; public long totalWakeups = 0; public long rootWakeups = 0; public long systemWakeups = 0; public long nonApplicationWakeups = 0; public long applicationWakeups = 0; public long unroutedWakeups = 0; public long durationSec = 0; public WakeupStats(String iface) { this.iface = iface; } /** Update durationSec with current time. */ public void updateDuration() { durationSec = (SystemClock.elapsedRealtime() - creationTimeMs) / 1000; } /** Update wakeup counters for the given WakeupEvent. */ public void countEvent(WakeupEvent ev) { totalWakeups++; switch (ev.uid) { case Process.ROOT_UID: rootWakeups++; break; case Process.SYSTEM_UID: systemWakeups++; break; case NO_UID: unroutedWakeups++; break; default: if (ev.uid >= Process.FIRST_APPLICATION_UID) { applicationWakeups++; } else { nonApplicationWakeups++; } break; } } @Override public String toString() { updateDuration(); return new StringBuilder() .append("WakeupStats(").append(iface) .append(", total: ").append(totalWakeups) .append(", root: ").append(rootWakeups) .append(", system: ").append(systemWakeups) .append(", apps: ").append(applicationWakeups) .append(", non-apps: ").append(nonApplicationWakeups) .append(", unrouted: ").append(unroutedWakeups) .append(", ").append(durationSec).append("s)") .toString(); } }
proto/src/ipconnectivity.proto +35 −0 Original line number Diff line number Diff line Loading @@ -473,6 +473,38 @@ message NetworkStats { repeated Pair validation_states = 8; } // Represents statistics from NFLOG wakeup events due to ingress packets. // Since oc-mr1. // Next tag: 8. message WakeupStats { // The time duration in seconds covered by these stats, for deriving // exact wakeup rates. optional int64 duration_sec = 1; // The total number of ingress packets waking up the device. optional int64 total_wakeups = 2; // The total number of wakeup packets routed to a socket belonging to // the root uid (uid 0). optional int64 root_wakeups = 3; // The total number of wakeup packets routed to a socket belonging to // the system server (uid 1000). optional int64 system_wakeups = 4; // The total number of wakeup packets routed to a socket belonging to // an application (uid > 9999). optional int64 application_wakeups = 5; // The total number of wakeup packets routed to a socket belonging to another // uid than the root uid, system uid, or an application uid (any uid in // between [1001, 9999]. See android.os.Process for possible uids. optional int64 non_application_wakeups = 6; // The total number of wakeup packets with no associated sockets. optional int64 unrouted_wakeups = 7; } // Represents one of the IP connectivity event defined in this file. // Next tag: 20 message IpConnectivityEvent { Loading Loading @@ -547,6 +579,9 @@ message IpConnectivityEvent { // Network statistics. NetworkStats network_stats = 19; // Ingress packet wakeup statistics. WakeupStats wakeup_stats = 20; }; }; Loading
services/core/java/com/android/server/connectivity/IpConnectivityEventBuilder.java +17 −0 Original line number Diff line number Diff line Loading @@ -38,6 +38,7 @@ import android.net.metrics.IpReachabilityEvent; import android.net.metrics.NetworkEvent; import android.net.metrics.RaEvent; import android.net.metrics.ValidationProbeEvent; import android.net.metrics.WakeupStats; import android.os.Parcelable; import android.util.SparseArray; import android.util.SparseIntArray; Loading Loading @@ -115,6 +116,22 @@ final public class IpConnectivityEventBuilder { return out; } public static IpConnectivityEvent toProto(WakeupStats in) { IpConnectivityLogClass.WakeupStats wakeupStats = new IpConnectivityLogClass.WakeupStats(); in.updateDuration(); wakeupStats.durationSec = in.durationSec; wakeupStats.totalWakeups = in.totalWakeups; wakeupStats.rootWakeups = in.rootWakeups; wakeupStats.systemWakeups = in.systemWakeups; wakeupStats.nonApplicationWakeups = in.nonApplicationWakeups; wakeupStats.applicationWakeups = in.applicationWakeups; wakeupStats.unroutedWakeups = in.unroutedWakeups; final IpConnectivityEvent out = buildEvent(0, 0, in.iface); out.setWakeupStats(wakeupStats); return out; } private static IpConnectivityEvent buildEvent(int netId, long transports, String ifname) { final IpConnectivityEvent ev = new IpConnectivityEvent(); ev.networkId = netId; Loading
services/core/java/com/android/server/connectivity/NetdEventListenerService.java +91 −7 Original line number Diff line number Diff line Loading @@ -16,6 +16,8 @@ package com.android.server.connectivity; import static android.util.TimeUtils.NANOS_PER_MS; import android.content.Context; import android.net.ConnectivityManager; import android.net.INetdEventCallback; Loading @@ -25,9 +27,12 @@ import android.net.metrics.ConnectStats; import android.net.metrics.DnsEvent; import android.net.metrics.INetdEventListener; import android.net.metrics.IpConnectivityLog; import android.net.metrics.WakeupEvent; import android.net.metrics.WakeupStats; import android.os.RemoteException; import android.text.format.DateUtils; import android.util.Log; import android.util.ArrayMap; import android.util.SparseArray; import com.android.internal.annotations.GuardedBy; import com.android.internal.annotations.VisibleForTesting; Loading Loading @@ -59,12 +64,28 @@ public class NetdEventListenerService extends INetdEventListener.Stub { private static final int CONNECT_LATENCY_FILL_RATE = 15 * (int) DateUtils.SECOND_IN_MILLIS; private static final int CONNECT_LATENCY_MAXIMUM_RECORDS = 20000; @VisibleForTesting static final int WAKEUP_EVENT_BUFFER_LENGTH = 1024; // TODO: dedup this String constant with the one used in // ConnectivityService#wakeupModifyInterface(). @VisibleForTesting static final String WAKEUP_EVENT_IFACE_PREFIX = "iface:"; // Sparse arrays of DNS and connect events, grouped by net id. @GuardedBy("this") private final SparseArray<DnsEvent> mDnsEvents = new SparseArray<>(); @GuardedBy("this") private final SparseArray<ConnectStats> mConnectEvents = new SparseArray<>(); // Array of aggregated wakeup event stats, grouped by interface name. @GuardedBy("this") private final ArrayMap<String, WakeupStats> mWakeupStats = new ArrayMap<>(); // Ring buffer array for storing packet wake up events sent by Netd. @GuardedBy("this") private final WakeupEvent[] mWakeupEvents = new WakeupEvent[WAKEUP_EVENT_BUFFER_LENGTH]; @GuardedBy("this") private long mWakeupEventCursor = 0; private final ConnectivityManager mCm; @GuardedBy("this") Loading Loading @@ -137,11 +158,62 @@ public class NetdEventListenerService extends INetdEventListener.Stub { @Override public synchronized void onWakeupEvent(String prefix, int uid, int gid, long timestampNs) { maybeVerboseLog("onWakeupEvent(%s, %d, %d, %sns)", prefix, uid, gid, timestampNs); // TODO: add ip protocol and port String iface = prefix.replaceFirst(WAKEUP_EVENT_IFACE_PREFIX, ""); final long timestampMs; if (timestampNs > 0) { timestampMs = timestampNs / NANOS_PER_MS; } else { timestampMs = System.currentTimeMillis(); } addWakupEvent(iface, timestampMs, uid); } @GuardedBy("this") private void addWakupEvent(String iface, long timestampMs, int uid) { int index = wakeupEventIndex(mWakeupEventCursor); mWakeupEventCursor++; WakeupEvent event = new WakeupEvent(); event.iface = iface; event.timestampMs = timestampMs; event.uid = uid; mWakeupEvents[index] = event; WakeupStats stats = mWakeupStats.get(iface); if (stats == null) { stats = new WakeupStats(iface); mWakeupStats.put(iface, stats); } stats.countEvent(event); } @GuardedBy("this") private WakeupEvent[] getWakeupEvents() { int length = (int) Math.min(mWakeupEventCursor, (long) mWakeupEvents.length); WakeupEvent[] out = new WakeupEvent[length]; // Reverse iteration from youngest event to oldest event. long inCursor = mWakeupEventCursor - 1; int outIdx = out.length - 1; while (outIdx >= 0) { out[outIdx--] = mWakeupEvents[wakeupEventIndex(inCursor--)]; } return out; } private static int wakeupEventIndex(long cursor) { return (int) Math.abs(cursor % WAKEUP_EVENT_BUFFER_LENGTH); } public synchronized void flushStatistics(List<IpConnectivityEvent> events) { flushProtos(events, mConnectEvents, IpConnectivityEventBuilder::toProto); flushProtos(events, mDnsEvents, IpConnectivityEventBuilder::toProto); for (int i = 0; i < mWakeupStats.size(); i++) { events.add(IpConnectivityEventBuilder.toProto(mWakeupStats.valueAt(i))); } mWakeupStats.clear(); } public synchronized void dump(PrintWriter writer) { Loading @@ -153,13 +225,22 @@ public class NetdEventListenerService extends INetdEventListener.Stub { } public synchronized void list(PrintWriter pw) { listEvents(pw, mConnectEvents, (x) -> x); listEvents(pw, mDnsEvents, (x) -> x); listEvents(pw, mConnectEvents, (x) -> x, "\n"); listEvents(pw, mDnsEvents, (x) -> x, "\n"); for (int i = 0; i < mWakeupStats.size(); i++) { pw.println(mWakeupStats.valueAt(i)); } for (WakeupEvent wakeup : getWakeupEvents()) { pw.println(wakeup); } } public synchronized void listAsProtos(PrintWriter pw) { listEvents(pw, mConnectEvents, IpConnectivityEventBuilder::toProto); listEvents(pw, mDnsEvents, IpConnectivityEventBuilder::toProto); listEvents(pw, mConnectEvents, IpConnectivityEventBuilder::toProto, ""); listEvents(pw, mDnsEvents, IpConnectivityEventBuilder::toProto, ""); for (int i = 0; i < mWakeupStats.size(); i++) { pw.print(IpConnectivityEventBuilder.toProto(mWakeupStats.valueAt(i))); } } private static <T> void flushProtos(List<IpConnectivityEvent> out, SparseArray<T> in, Loading @@ -170,10 +251,13 @@ public class NetdEventListenerService extends INetdEventListener.Stub { in.clear(); } public static <T> void listEvents( PrintWriter pw, SparseArray<T> events, Function<T, Object> mapper) { private static <T> void listEvents( PrintWriter pw, SparseArray<T> events, Function<T, Object> mapper, String separator) { // Proto derived Classes have toString method that adds a \n at the end. // Let the caller control that by passing in the line separator explicitly. for (int i = 0; i < events.size(); i++) { pw.println(mapper.apply(events.valueAt(i)).toString()); pw.print(mapper.apply(events.valueAt(i))); pw.print(separator); } } Loading