Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 1a1a7a00 authored by Pablo Gamito's avatar Pablo Gamito
Browse files

Add limits to ProtoLog interned data in Perfetto traces. 

This change introduces thresholds for the number of interned messages and strings, as well as the total size of interned strings. When these limits are reached, the incremental state of the ProtoLog data source is reset, ensuring that the trace data does not grow excessively large during long tracing sessions.

Bug: 420546420
Test: atest TracingTests
Flag: EXEMPT minor bug fix
Change-Id: I93e5b2bc169490614fab02c7c1e95405abe3e1b0
parent 91f3a2db

core/java/com/android/internal/protolog/PerfettoProtoLogImpl.java

+28 −2
Original line number Diff line number Diff line
@@ -98,6 +98,8 @@ public abstract class PerfettoProtoLogImpl extends IProtoLogClient.Stub implemen 
        TracingInstanceStartCallback, TracingInstanceStopCallback, TracingFlushCallback {

    private static final String LOG_TAG = "ProtoLog";

    public static final String NULL_STRING = "null";

    @VisibleForTesting

    public static int MAX_INTERNED_STRINGS_SIZE_BYTES_BEFORE_RESET = 4 * 1024 * 1024; // 4MB

    private final AtomicInteger mTracingInstances = new AtomicInteger();



    @NonNull
@@ -492,7 +494,8 @@ public abstract class PerfettoProtoLogImpl extends IProtoLogClient.Stub implemen 
        if (isProtoEnabled()) {

            long tsNanos = SystemClock.elapsedRealtimeNanos();

            final String stacktrace;

            if (mCollectStackTraceGroupCounts.getOrDefault(group.name(), 0) > 0) {

            if (logLevel == LogLevel.WTF

                    || mCollectStackTraceGroupCounts.getOrDefault(group.name(), 0) > 0) {

                stacktrace = collectStackTrace();

            } else {

                stacktrace = null;
@@ -725,6 +728,18 @@ public abstract class PerfettoProtoLogImpl extends IProtoLogClient.Stub implemen 
            @NonNull IProtoLogGroup logGroup, @NonNull String message) {

        final ProtoLogDataSource.IncrementalState incrementalState = ctx.getIncrementalState();



        final Long messageHash = hash(level, logGroup.name(), message);



        if (android.tracing.Flags.protologAutoClearIncrementalState()

                && !incrementalState.protologMessageInterningSet.contains(messageHash)) {

            final boolean sizeThresholdReached =

                    incrementalState.internedStringsSizeBytes + message.length()

                            >= MAX_INTERNED_STRINGS_SIZE_BYTES_BEFORE_RESET;

            if (sizeThresholdReached) {

                incrementalState.reset();

            }

        }



        if (!incrementalState.clearReported) {

            final ProtoOutputStream os = ctx.newTracePacket(8);

            os.write(SEQUENCE_FLAGS, SEQ_INCREMENTAL_STATE_CLEARED);
@@ -747,9 +762,9 @@ public abstract class PerfettoProtoLogImpl extends IProtoLogClient.Stub implemen 
            os.end(protologViewerConfigToken);

        }



        final Long messageHash = hash(level, logGroup.name(), message);

        if (!incrementalState.protologMessageInterningSet.contains(messageHash)) {

            incrementalState.protologMessageInterningSet.add(messageHash);

            incrementalState.internedStringsSizeBytes += message.length();



            final ProtoOutputStream os = ctx.newTracePacket(128);
@@ -824,6 +839,16 @@ public abstract class PerfettoProtoLogImpl extends IProtoLogClient.Stub implemen 
    ) {

        final ProtoLogDataSource.IncrementalState incrementalState = ctx.getIncrementalState();



        if (android.tracing.Flags.protologAutoClearIncrementalState()

                && !internMap.containsKey(string)) {

            final boolean sizeThresholdReached =

                    incrementalState.internedStringsSizeBytes + string.length()

                            >= MAX_INTERNED_STRINGS_SIZE_BYTES_BEFORE_RESET;

            if (sizeThresholdReached) {

                incrementalState.reset();

            }

        }



        if (!incrementalState.clearReported) {

            final ProtoOutputStream os = ctx.newTracePacket(8);

            os.write(SEQUENCE_FLAGS, SEQ_INCREMENTAL_STATE_CLEARED);
@@ -833,6 +858,7 @@ public abstract class PerfettoProtoLogImpl extends IProtoLogClient.Stub implemen 
        if (!internMap.containsKey(string)) {

            final int internedIndex = internMap.size() + 1;

            internMap.put(string, internedIndex);

            incrementalState.internedStringsSizeBytes += string.length();



            final ProtoOutputStream os = ctx.newTracePacket(64);

            final long token = os.start(INTERNED_DATA);

core/java/com/android/internal/protolog/ProtoLogDataSource.java

+14 −0
Original line number Diff line number Diff line
@@ -237,7 +237,21 @@ public class ProtoLogDataSource extends DataSource<ProtoLogDataSource.Instance, 
        public final Set<Long> protologMessageInterningSet = new HashSet<>();

        public final Map<String, Integer> argumentInterningMap = new HashMap<>();

        public final Map<String, Integer> stacktraceInterningMap = new HashMap<>();

        public long internedStringsSizeBytes = 0;

        public boolean clearReported = false;



        /**

         * Resets the incremental state, clearing all interned data. This is used to prevent

         * the state from growing too large during long tracing sessions.

         */

        public void reset() {

            protologGroupInterningSet.clear();

            protologMessageInterningSet.clear();

            argumentInterningMap.clear();

            stacktraceInterningMap.clear();

            internedStringsSizeBytes = 0;

            clearReported = false;

        }

    }



    public static class ProtoLogConfig {

tests/Tracing/src/com/android/internal/protolog/ProcessedPerfettoProtoLogImplTest.java

+48 −3
Original line number Diff line number Diff line
@@ -29,9 +29,11 @@ import static org.mockito.ArgumentMatchers.anyLong; 
import static org.mockito.ArgumentMatchers.eq;

import static org.mockito.Mockito.doAnswer;

import static org.mockito.Mockito.never;

import static org.junit.Assume.assumeTrue;

import static org.mockito.Mockito.verify;

import static org.mockito.Mockito.when;



import static perfetto.protos.TracePacketOuterClass.TracePacket.SequenceFlags.SEQ_INCREMENTAL_STATE_CLEARED;

import static perfetto.protos.TracePacketOuterClass.TracePacket.SequenceFlags.SEQ_NEEDS_INCREMENTAL_STATE;



import android.os.SystemClock;
@@ -97,6 +99,9 @@ public class ProcessedPerfettoProtoLogImplTest { 


    private static ProtoLogViewerConfigReader sReader;



    private static int sOriginalMaxInternedStringsSize;





    public ProcessedPerfettoProtoLogImplTest() throws IOException { }



    @BeforeClass
@@ -186,6 +191,9 @@ public class ProcessedPerfettoProtoLogImplTest { 
                (instance) -> sCacheUpdater.update(instance), TestProtoLogGroup.values(),

                sProtoLogConfigurationService);

        sProtoLog.enable();



        sOriginalMaxInternedStringsSize =

                PerfettoProtoLogImpl.MAX_INTERNED_STRINGS_SIZE_BYTES_BEFORE_RESET;

    }



    @Before
@@ -198,6 +206,8 @@ public class ProcessedPerfettoProtoLogImplTest { 
    @After

    public void tearDown() {

        ProtoLogImpl.setSingleInstance(null);

        PerfettoProtoLogImpl.MAX_INTERNED_STRINGS_SIZE_BYTES_BEFORE_RESET =

                sOriginalMaxInternedStringsSize;

    }



    @Test
@@ -562,9 +572,6 @@ public class ProcessedPerfettoProtoLogImplTest { 
        } finally {

            traceMonitor.stop(writer);

        }



        final ResultReader reader = new ResultReader(writer.write());

        assertThrows(java.net.SocketException.class, reader::readProtoLogTrace);

    }



    @Test
@@ -1309,6 +1316,44 @@ public class ProcessedPerfettoProtoLogImplTest { 
                .that(sequenceFlag & incrementalStateFlag).isEqualTo(incrementalStateFlag);

    }



    @Test

    public void incrementalStateResetWhenStringsTooLarge() throws IOException {

        assumeTrue(android.tracing.Flags.protologAutoClearIncrementalState());

        PerfettoProtoLogImpl.MAX_INTERNED_STRINGS_SIZE_BYTES_BEFORE_RESET = 10;



        PerfettoTraceMonitor traceMonitor = PerfettoTraceMonitor.newBuilder()

                .enableProtoLog(true, List.of(), TEST_PROTOLOG_DATASOURCE_NAME)

                .build();



        final var writer = createTempWriter(mTracingDirectory);

        try {

            traceMonitor.start();



            // Log a 5-char string, should not reset yet.

            sProtoLog.log(LogLevel.DEBUG, TestProtoLogGroup.TEST_GROUP, 6,

                    LogDataType.STRING, new Object[]{"12345"});



            // Log another 4-char string, should not reset yet (5 + 4 = 9).

            sProtoLog.log(LogLevel.DEBUG, TestProtoLogGroup.TEST_GROUP, 6,

                    LogDataType.STRING, new Object[]{"abcd"});



            // Log a message string, should trigger reset because 9 + message_len > 10.

            sProtoLog.log(LogLevel.DEBUG, TestProtoLogGroup.TEST_GROUP, "message");

        } finally {

            traceMonitor.stop(writer);

        }



        final ResultReader reader = new ResultReader(writer.write());

        final var traceBytes = reader.readBytes(TraceType.PERFETTO, Tag.ALL);

        final var trace = Trace.parseFrom(traceBytes);



        final var clearPackets = trace.getPacketList().stream()

                .filter(it -> (it.getSequenceFlags()

                        & SEQ_INCREMENTAL_STATE_CLEARED.getNumber()) != 0)

                .toList();

        Truth.assertThat(clearPackets).hasSize(1);

    }



    private enum TestProtoLogGroup implements IProtoLogGroup {

        TEST_GROUP(true, false, "TEST_TAG");