Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit bedb2008 authored by Harshit Mahajan's avatar Harshit Mahajan
Browse files

Update logs for CrashRecovery Module 

Stop using PackageManagerServiceUtils#logCriticalInfo as we move to
module.
Introduce "crashrecovery-events.txt" in "/data/system" to store the
recovery events and persist the events across the boots. We would also
be dumping this events from PackageWatchdog#dump to help in debuggging.

Bug: 289203818
Test: adb shell dumpsys rollback; adb bugreport
Flag: EXEMPT logging changes only
Change-Id: I427fa1fdf8a7467ffae16a314b855698bb41b9a9
parent 9a6338cd

services/core/java/com/android/server/PackageWatchdog.java

+14 −8
Original line number Diff line number Diff line
@@ -20,6 +20,8 @@ import static android.content.Intent.ACTION_REBOOT; 
import static android.content.Intent.ACTION_SHUTDOWN;

import static android.service.watchdog.ExplicitHealthCheckService.PackageConfig;



import static com.android.server.crashrecovery.CrashRecoveryUtils.dumpCrashRecoveryEvents;



import static java.lang.annotation.RetentionPolicy.SOURCE;



import android.annotation.IntDef;
@@ -44,6 +46,7 @@ import android.text.TextUtils; 
import android.util.ArrayMap;

import android.util.ArraySet;

import android.util.AtomicFile;

import android.util.IndentingPrintWriter;

import android.util.LongArrayQueue;

import android.util.Slog;

import android.util.Xml;
@@ -51,7 +54,6 @@ import android.util.Xml; 
import com.android.internal.annotations.GuardedBy;

import com.android.internal.annotations.VisibleForTesting;

import com.android.internal.os.BackgroundThread;

import com.android.internal.util.IndentingPrintWriter;

import com.android.internal.util.XmlUtils;

import com.android.modules.utils.TypedXmlPullParser;

import com.android.modules.utils.TypedXmlSerializer;
@@ -72,6 +74,7 @@ import java.io.IOException; 
import java.io.InputStream;

import java.io.ObjectInputStream;

import java.io.ObjectOutputStream;

import java.io.PrintWriter;

import java.lang.annotation.Retention;

import java.lang.annotation.RetentionPolicy;

import java.util.ArrayList;
@@ -1265,18 +1268,21 @@ public class PackageWatchdog { 




    /** Dump status of every observer in mAllObservers. */

    public void dump(IndentingPrintWriter pw) {

        pw.println("Package Watchdog status");

        pw.increaseIndent();

    public void dump(PrintWriter pw) {

        IndentingPrintWriter ipw = new IndentingPrintWriter(pw, "  ");

        ipw.println("Package Watchdog status");

        ipw.increaseIndent();

        synchronized (mLock) {

            for (String observerName : mAllObservers.keySet()) {

                pw.println("Observer name: " + observerName);

                pw.increaseIndent();

                ipw.println("Observer name: " + observerName);

                ipw.increaseIndent();

                ObserverInternal observerInternal = mAllObservers.get(observerName);

                observerInternal.dump(pw);

                pw.decreaseIndent();

                observerInternal.dump(ipw);

                ipw.decreaseIndent();

            }

        }

        ipw.decreaseIndent();

        dumpCrashRecoveryEvents(ipw);

    }



    @VisibleForTesting

services/core/java/com/android/server/RescueParty.java

+5 −5
Original line number Diff line number Diff line
@@ -18,7 +18,7 @@ package com.android.server; 


import static android.provider.DeviceConfig.Properties;



import static com.android.server.pm.PackageManagerServiceUtils.logCriticalInfo;

import static com.android.server.crashrecovery.CrashRecoveryUtils.logCrashRecoveryEvent;



import android.annotation.IntDef;

import android.annotation.NonNull;
@@ -291,13 +291,13 @@ public class RescueParty { 
                Properties properties = new Properties.Builder(namespaceToReset).build();

                try {

                    if (!DeviceConfig.setProperties(properties)) {

                        logCriticalInfo(Log.ERROR, "Failed to clear properties under "

                        logCrashRecoveryEvent(Log.ERROR, "Failed to clear properties under "

                            + namespaceToReset

                            + ". Running `device_config get_sync_disabled_for_tests` will confirm"

                            + " if config-bulk-update is enabled.");

                    }

                } catch (DeviceConfig.BadConfigException exception) {

                    logCriticalInfo(Log.WARN, "namespace " + namespaceToReset

                    logCrashRecoveryEvent(Log.WARN, "namespace " + namespaceToReset

                            + " is already banned, skip reset.");

                }

            }
@@ -528,7 +528,7 @@ public class RescueParty { 
            if (!TextUtils.isEmpty(failedPackage)) {

                successMsg += " for package " + failedPackage;

            }

            logCriticalInfo(Log.DEBUG, successMsg);

            logCrashRecoveryEvent(Log.DEBUG, successMsg);

        } catch (Throwable t) {

            logRescueException(level, failedPackage, t);

        }
@@ -687,7 +687,7 @@ public class RescueParty { 
        if (!TextUtils.isEmpty(failedPackageName)) {

            failureMsg += " for package " + failedPackageName;

        }

        logCriticalInfo(Log.ERROR, failureMsg + ": " + msg);

        logCrashRecoveryEvent(Log.ERROR, failureMsg + ": " + msg);

    }



    private static int mapRescueLevelToUserImpact(int rescueLevel) {

services/core/java/com/android/server/crashrecovery/CrashRecoveryUtils.java

0 → 100644
+85 −0
Original line number Diff line number Diff line 
/*

 * Copyright (C) 2024 The Android Open Source Project

 *

 * Licensed under the Apache License, Version 2.0 (the "License");

 * you may not use this file except in compliance with the License.

 * You may obtain a copy of the License at

 *

 *      http://www.apache.org/licenses/LICENSE-2.0

 *

 * Unless required by applicable law or agreed to in writing, software

 * distributed under the License is distributed on an "AS IS" BASIS,

 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

 * See the License for the specific language governing permissions and

 * limitations under the License.

 */



package com.android.server.crashrecovery;



import android.os.Environment;

import android.util.IndentingPrintWriter;

import android.util.Slog;



import java.io.BufferedReader;

import java.io.File;

import java.io.FileOutputStream;

import java.io.FileReader;

import java.io.IOException;

import java.io.PrintWriter;

import java.time.LocalDateTime;

import java.time.ZoneId;



/**

 * Class containing helper methods for the CrashRecoveryModule.

 *

 * @hide

 */

public class CrashRecoveryUtils {

    private static final String TAG = "CrashRecoveryUtils";

    private static final long MAX_CRITICAL_INFO_DUMP_SIZE = 1000 * 1000; // ~1MB

    private static final Object sFileLock = new Object();



    /** Persist recovery related events in crashrecovery events file.**/

    public static void logCrashRecoveryEvent(int priority, String msg) {

        Slog.println(priority, TAG, msg);

        try {

            File fname = getCrashRecoveryEventsFile();

            synchronized (sFileLock) {

                FileOutputStream out = new FileOutputStream(fname, true);

                PrintWriter pw = new PrintWriter(out);

                String dateString = LocalDateTime.now(ZoneId.systemDefault()).toString();

                pw.println(dateString + ": " + msg);

                pw.close();

            }

        } catch (IOException e) {

            Slog.e(TAG, "Unable to log CrashRecoveryEvents " + e.getMessage());

        }

    }



    /** Dump recovery related events from crashrecovery events file.**/

    public static void dumpCrashRecoveryEvents(IndentingPrintWriter pw) {

        pw.println("CrashRecovery Events: ");

        pw.increaseIndent();

        final File file = getCrashRecoveryEventsFile();

        final long skipSize = file.length() - MAX_CRITICAL_INFO_DUMP_SIZE;

        synchronized (sFileLock) {

            try (BufferedReader in = new BufferedReader(new FileReader(file))) {

                if (skipSize > 0) {

                    in.skip(skipSize);

                }

                String line;

                while ((line = in.readLine()) != null) {

                    pw.println(line);

                }

            } catch (IOException e) {

                Slog.e(TAG, "Unable to dump CrashRecoveryEvents " + e.getMessage());

            }

        }

        pw.decreaseIndent();

    }



    private static File getCrashRecoveryEventsFile() {

        File systemDir = new File(Environment.getDataDirectory(), "system");

        return new File(systemDir, "crashrecovery-events.txt");

    }

}

services/core/java/com/android/server/rollback/RollbackPackageHealthObserver.java

+8 −2
Original line number Diff line number Diff line
@@ -18,6 +18,8 @@ package com.android.server.rollback; 


import static android.content.pm.Flags.provideInfoOfApkInApex;



import static com.android.server.crashrecovery.CrashRecoveryUtils.logCrashRecoveryEvent;



import android.annotation.AnyThread;

import android.annotation.NonNull;

import android.annotation.Nullable;
@@ -40,6 +42,7 @@ import android.os.PowerManager; 
import android.os.SystemProperties;

import android.sysprop.CrashRecoveryProperties;

import android.util.ArraySet;

import android.util.Log;

import android.util.Slog;

import android.util.SparseArray;
@@ -532,11 +535,13 @@ public final class RollbackPackageHealthObserver implements PackageHealthObserve 
    private void rollbackPackage(RollbackInfo rollback, VersionedPackage failedPackage,

            @FailureReasons int rollbackReason) {

        assertInWorkerThread();

        String failedPackageName = (failedPackage == null ? null : failedPackage.getPackageName());



        Slog.i(TAG, "Rolling back package. RollbackId: " + rollback.getRollbackId()

                + " failedPackage: "

                + (failedPackage == null ? null : failedPackage.getPackageName())

                + " failedPackage: " + failedPackageName

                + " rollbackReason: " + rollbackReason);

        logCrashRecoveryEvent(Log.DEBUG, String.format("Rolling back %s. Reason: %s",

                failedPackageName, rollbackReason));

        final RollbackManager rollbackManager = mContext.getSystemService(RollbackManager.class);

        int reasonToLog = WatchdogRollbackLogger.mapFailureReasonToMetric(rollbackReason);

        final String failedPackageToLog;
@@ -724,6 +729,7 @@ public final class RollbackPackageHealthObserver implements PackageHealthObserve 
        }



        Slog.i(TAG, "Rolling back all available low impact rollbacks");

        logCrashRecoveryEvent(Log.DEBUG, "Rolling back all available. Reason: " + rollbackReason);

        // Add all rollback ids to mPendingStagedRollbackIds, so that we do not reboot before all

        // pending staged rollbacks are handled.

        for (RollbackInfo rollback : lowImpactRollbacks) {

services/tests/mockingservicestests/src/com/android/server/crashrecovery/CrashRecoveryUtilsTest.java

0 → 100644
+145 −0
Original line number Diff line number Diff line 
/*

 * Copyright (C) 2024 The Android Open Source Project

 *

 * Licensed under the Apache License, Version 2.0 (the "License");

 * you may not use this file except in compliance with the License.

 * You may obtain a copy of the License at

 *

 *      http://www.apache.org/licenses/LICENSE-2.0

 *

 * Unless required by applicable law or agreed to in writing, software

 * distributed under the License is distributed on an "AS IS" BASIS,

 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

 * See the License for the specific language governing permissions and

 * limitations under the License.

 */



package com.android.server.crashrecovery;







import static com.google.common.truth.Truth.assertThat;



import static org.mockito.quality.Strictness.LENIENT;

import static org.junit.Assert.fail;



import android.content.Context;

import android.os.Environment;

import android.util.IndentingPrintWriter;

import android.util.Log;



import androidx.test.core.app.ApplicationProvider;

import androidx.test.runner.AndroidJUnit4;



import com.android.dx.mockito.inline.extended.ExtendedMockito;



import org.junit.After;

import org.junit.Before;

import org.junit.Test;

import org.junit.runner.RunWith;

import org.mockito.MockitoSession;



import java.io.File;

import java.io.FileInputStream;

import java.io.IOException;

import java.io.StringWriter;

import java.nio.charset.StandardCharsets;

import java.nio.file.Files;





/**

 * Test CrashRecovery Utils.

 */

@RunWith(AndroidJUnit4.class)

public class CrashRecoveryUtilsTest {



    private MockitoSession mStaticMockSession;

    private final String mLogMsg = "Logging from test";

    private final String mCrashrecoveryEventTag = "CrashRecovery Events: ";

    private File mCacheDir;



    @Before

    public void setup() throws IOException {

        Context context = ApplicationProvider.getApplicationContext();

        mCacheDir = context.getCacheDir();

        mStaticMockSession = ExtendedMockito.mockitoSession()

                .spyStatic(Environment.class)

                .strictness(LENIENT)

                .startMocking();

        ExtendedMockito.doReturn(mCacheDir).when(() -> Environment.getDataDirectory());



        createCrashRecoveryEventsTempDir();

    }



    @After

    public void tearDown() throws IOException {

        mStaticMockSession.finishMocking();

        deleteCrashRecoveryEventsTempFile();

    }



    @Test

    public void testCrashRecoveryUtils() {

        testLogCrashRecoveryEvent();

        testDumpCrashRecoveryEvents();

    }



    @Test

    public void testDumpCrashRecoveryEventsWithoutAnyLogs() {

        assertThat(getCrashRecoveryEventsTempFile().exists()).isFalse();

        StringWriter sw = new StringWriter();

        IndentingPrintWriter ipw = new IndentingPrintWriter(sw, "  ");

        CrashRecoveryUtils.dumpCrashRecoveryEvents(ipw);

        ipw.close();



        String dump = sw.getBuffer().toString();

        assertThat(dump).contains(mCrashrecoveryEventTag);

        assertThat(dump).doesNotContain(mLogMsg);

    }



    private void testLogCrashRecoveryEvent() {

        assertThat(getCrashRecoveryEventsTempFile().exists()).isFalse();

        CrashRecoveryUtils.logCrashRecoveryEvent(Log.WARN, mLogMsg);



        assertThat(getCrashRecoveryEventsTempFile().exists()).isTrue();

        String fileContent = null;

        try {

            File file = getCrashRecoveryEventsTempFile();

            FileInputStream fis = new FileInputStream(file);

            byte[] data = new byte[(int) file.length()];

            fis.read(data);

            fis.close();

            fileContent = new String(data, StandardCharsets.UTF_8);

        } catch (Exception e) {

            fail("Unable to read the events file");

        }

        assertThat(fileContent).contains(mLogMsg);

    }



    private void testDumpCrashRecoveryEvents() {

        StringWriter sw = new StringWriter();

        IndentingPrintWriter ipw = new IndentingPrintWriter(sw, "  ");

        CrashRecoveryUtils.dumpCrashRecoveryEvents(ipw);

        ipw.close();



        String dump = sw.getBuffer().toString();

        assertThat(dump).contains(mCrashrecoveryEventTag);

        assertThat(dump).contains(mLogMsg);

    }



    private void createCrashRecoveryEventsTempDir() throws IOException {

        Files.deleteIfExists(getCrashRecoveryEventsTempFile().toPath());

        File mMockDirectory = new File(mCacheDir, "system");

        if (!mMockDirectory.exists()) {

            assertThat(mMockDirectory.mkdir()).isTrue();

        }

    }



    private void deleteCrashRecoveryEventsTempFile() throws IOException {

        Files.deleteIfExists(getCrashRecoveryEventsTempFile().toPath());

    }



    private File getCrashRecoveryEventsTempFile() {

        File systemTempDir = new File(mCacheDir, "system");

        return new File(systemTempDir, "crashrecovery-events.txt");

    }

}