Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 3c56021d authored by Shrinidhi Hegde's avatar Shrinidhi Hegde
Browse files

update package watchdog to perform mitigations after every reboot once threshold is reached. 

Since the bootloop is not for complete boot rather for  system-server
restarts, we dont have to be as conscious about performing higher
impact mitigations and we can perform them earlier. (Reboots
initiated by user doesnt trigger a bootloop scenario.)

Removed the increment boot count threshold needed between 2 mitigations.
Also removed BOOT_LOOP_THRESHOLD(default value: 15) which was needed to
perform any high impact mitigations.

PackageWatchdog would wait for DEFAULT_BOOT_LOOP_TRIGGER_COUNT (default
value: 5) after a reboot (ex: WARM_REBOOT step in rescue party, mainline
rollback). Changed the behaviour to wait for DEFAULT_BOOT_LOOP_TRIGGER_COUNT
only for first mitigation.

MitigationCount was not getting reset properly after de-escalation
window. Updated the logic to save mitigationCount to the file after reset.

Did manual tests by killing system_server process using the following
command:
`adb shell kill -9 $(adb shell pidof system_server)`
Verified that the first mitigation was performed at 5 restarts.
Every restart after that resulted in next mitigation being executed.

Ran `atest PackageEatcdogTest` to verify PackageWatchdogTest and
CrashRecoveryTest changes.

Test: Unit test, manual test
Bug: 291137901
Change-Id: Ia0174e0e5dca09c5d4e9d3a12909c5e5087b4dbf
parent 417dae88

packages/CrashRecovery/services/java/com/android/server/PackageWatchdog.java

+31 −45
Original line number Diff line number Diff line
@@ -137,17 +137,6 @@ public class PackageWatchdog { 
    static final int DEFAULT_BOOT_LOOP_TRIGGER_COUNT = 5;



    static final long DEFAULT_BOOT_LOOP_TRIGGER_WINDOW_MS = TimeUnit.MINUTES.toMillis(10);

    // Boot loop at which packageWatchdog starts first mitigation

    private static final String BOOT_LOOP_THRESHOLD =

            "persist.device_config.configuration.boot_loop_threshold";

    @VisibleForTesting

    static final int DEFAULT_BOOT_LOOP_THRESHOLD = 15;

    // Once boot_loop_threshold is surpassed next mitigation would be triggered after

    // specified number of reboots.

    private static final String BOOT_LOOP_MITIGATION_INCREMENT =

            "persist.device_config.configuration..boot_loop_mitigation_increment";

    @VisibleForTesting

    static final int DEFAULT_BOOT_LOOP_MITIGATION_INCREMENT = 2;



    // Threshold level at which or above user might experience significant disruption.

    private static final String MAJOR_USER_IMPACT_LEVEL_THRESHOLD =
@@ -253,15 +242,8 @@ public class PackageWatchdog { 
        mConnectivityModuleConnector = connectivityModuleConnector;

        mSystemClock = clock;

        mNumberOfNativeCrashPollsRemaining = NUMBER_OF_NATIVE_CRASH_POLLS;

        if (Flags.recoverabilityDetection()) {

            mBootThreshold = new BootThreshold(DEFAULT_BOOT_LOOP_TRIGGER_COUNT,

                    DEFAULT_BOOT_LOOP_TRIGGER_WINDOW_MS,

                    SystemProperties.getInt(BOOT_LOOP_MITIGATION_INCREMENT,

                            DEFAULT_BOOT_LOOP_MITIGATION_INCREMENT));

        } else {

        mBootThreshold = new BootThreshold(DEFAULT_BOOT_LOOP_TRIGGER_COUNT,

                DEFAULT_BOOT_LOOP_TRIGGER_WINDOW_MS);

        }



        loadFromFile();

        sPackageWatchdog = this;
@@ -526,10 +508,16 @@ public class PackageWatchdog { 
    /**

     * Called when the system server boots. If the system server is detected to be in a boot loop,

     * query each observer and perform the mitigation action with the lowest user impact.

     *

     * Note: PackageWatchdog considers system_server restart loop as bootloop. Full reboots

     * are not counted in bootloop.

     */

    @SuppressWarnings("GuardedBy")

    public void noteBoot() {

        synchronized (mLock) {

            // if boot count has reached threshold, start mitigation.

            // We wait until threshold number of restarts only for the first time. Perform

            // mitigations for every restart after that.

            boolean mitigate = mBootThreshold.incrementAndTest();

            if (mitigate) {

                if (!Flags.recoverabilityDetection()) {
@@ -557,9 +545,6 @@ public class PackageWatchdog { 
                }

                if (currentObserverToNotify != null) {

                    if (Flags.recoverabilityDetection()) {

                        if (currentObserverImpact < getUserImpactLevelLimit()

                                || (currentObserverImpact >= getUserImpactLevelLimit()

                                        && mBootThreshold.getCount() >= getBootLoopThreshold())) {

                        int currentObserverMitigationCount =

                                currentObserverInternal.getBootMitigationCount() + 1;

                        currentObserverInternal.setBootMitigationCount(
@@ -567,7 +552,6 @@ public class PackageWatchdog { 
                        saveAllObserversBootMitigationCountToMetadata(METADATA_FILE);

                        currentObserverToNotify.executeBootLoopMitigation(

                                currentObserverMitigationCount);

                        }

                    } else {

                        mBootThreshold.setMitigationCount(mitigationCount);

                        mBootThreshold.saveMitigationCountToMetadata();
@@ -647,11 +631,6 @@ public class PackageWatchdog { 
                DEFAULT_MAJOR_USER_IMPACT_LEVEL_THRESHOLD);

    }



    private int getBootLoopThreshold() {

        return SystemProperties.getInt(BOOT_LOOP_THRESHOLD,

                DEFAULT_BOOT_LOOP_THRESHOLD);

    }



    /** Possible severity values of the user impact of a {@link PackageHealthObserver#execute}. */

    @Retention(SOURCE)

    @IntDef(value = {PackageHealthObserverImpact.USER_IMPACT_LEVEL_0,
@@ -1827,16 +1806,10 @@ public class PackageWatchdog { 


        private final int mBootTriggerCount;

        private final long mTriggerWindow;

        private final int mBootMitigationIncrement;



        BootThreshold(int bootTriggerCount, long triggerWindow) {

            this(bootTriggerCount, triggerWindow, /*bootMitigationIncrement=*/ 1);

        }



        BootThreshold(int bootTriggerCount, long triggerWindow, int bootMitigationIncrement) {

            this.mBootTriggerCount = bootTriggerCount;

            this.mTriggerWindow = triggerWindow;

            this.mBootMitigationIncrement = bootMitigationIncrement;

        }



        public void reset() {
@@ -1915,6 +1888,7 @@ public class PackageWatchdog { 
            } else {

                readMitigationCountFromMetadataIfNecessary();

            }



            final long now = mSystemClock.uptimeMillis();

            if (now - getStart() < 0) {

                Slog.e(TAG, "Window was less than zero. Resetting start to current time.");
@@ -1939,20 +1913,32 @@ public class PackageWatchdog { 
                setCount(count);

                EventLogTags.writeRescueNote(Process.ROOT_UID, count, window);

                if (Flags.recoverabilityDetection()) {

                    boolean mitigate = (count >= mBootTriggerCount)

                            && (count - mBootTriggerCount) % mBootMitigationIncrement == 0;

                    return mitigate;

                    // After a reboot (e.g. by WARM_REBOOT or mainline rollback) we apply

                    // mitigations without waiting for DEFAULT_BOOT_LOOP_TRIGGER_COUNT.

                    return (count >= mBootTriggerCount)

                            || (performedMitigationsDuringWindow() && count > 1);

                }

                return count >= mBootTriggerCount;

            }

        }



        @GuardedBy("mLock")

        private boolean performedMitigationsDuringWindow() {

            for (ObserverInternal observerInternal: mAllObservers.values()) {

                if (observerInternal.getBootMitigationCount() > 0) {

                    return true;

                }

            }

            return false;

        }



        @GuardedBy("mLock")

        private void resetAllObserversBootMitigationCount() {

            for (int i = 0; i < mAllObservers.size(); i++) {

                final ObserverInternal observer = mAllObservers.valueAt(i);

                observer.setBootMitigationCount(0);

            }

            saveAllObserversBootMitigationCountToMetadata(METADATA_FILE);

        }



        @GuardedBy("mLock")

services/tests/mockingservicestests/src/com/android/server/RescuePartyTest.java

+1 −8
Original line number Diff line number Diff line
@@ -1111,16 +1111,9 @@ public class RescuePartyTest { 


        // mock properties in BootThreshold

        try {

            if (Flags.recoverabilityDetection()) {

                mSpyBootThreshold = spy(watchdog.new BootThreshold(

                    PackageWatchdog.DEFAULT_BOOT_LOOP_TRIGGER_COUNT,

                    PackageWatchdog.DEFAULT_BOOT_LOOP_TRIGGER_WINDOW_MS,

                    PackageWatchdog.DEFAULT_BOOT_LOOP_MITIGATION_INCREMENT));

            } else {

            mSpyBootThreshold = spy(watchdog.new BootThreshold(

                    PackageWatchdog.DEFAULT_BOOT_LOOP_TRIGGER_COUNT,

                    PackageWatchdog.DEFAULT_BOOT_LOOP_TRIGGER_WINDOW_MS));

            }

            mCrashRecoveryPropertiesMap = new HashMap<>();



            doAnswer((Answer<Integer>) invocationOnMock -> {

tests/PackageWatchdog/src/com/android/server/CrashRecoveryTest.java

+54 −62
Original line number Diff line number Diff line
@@ -66,6 +66,7 @@ import org.mockito.Answers; 
import org.mockito.ArgumentCaptor;

import org.mockito.Captor;

import org.mockito.Mock;

import org.mockito.Mockito;

import org.mockito.MockitoAnnotations;

import org.mockito.MockitoSession;

import org.mockito.quality.Strictness;
@@ -220,43 +221,36 @@ public class CrashRecoveryTest { 
        RescuePartyObserver rescuePartyObserver = setUpRescuePartyObserver(watchdog);



        verify(rescuePartyObserver, never()).executeBootLoopMitigation(1);

        int bootCounter = 0;



        for (int i = 0; i < PackageWatchdog.DEFAULT_BOOT_LOOP_TRIGGER_COUNT; i++) {

            watchdog.noteBoot();

            bootCounter += 1;

        }



        verify(rescuePartyObserver).executeBootLoopMitigation(1);

        verify(rescuePartyObserver, never()).executeBootLoopMitigation(2);



        for (int i = 0; i < PackageWatchdog.DEFAULT_BOOT_LOOP_MITIGATION_INCREMENT; i++) {

        watchdog.noteBoot();

            bootCounter += 1;

        }



        verify(rescuePartyObserver).executeBootLoopMitigation(2);

        verify(rescuePartyObserver, never()).executeBootLoopMitigation(3);



        int bootLoopThreshold = PackageWatchdog.DEFAULT_BOOT_LOOP_THRESHOLD - bootCounter;

        for (int i = 0; i < bootLoopThreshold; i++) {

        watchdog.noteBoot();

        }



        verify(rescuePartyObserver).executeBootLoopMitigation(3);

        verify(rescuePartyObserver, never()).executeBootLoopMitigation(4);



        for (int i = 0; i < PackageWatchdog.DEFAULT_BOOT_LOOP_MITIGATION_INCREMENT; i++) {

        watchdog.noteBoot();

        }



        verify(rescuePartyObserver).executeBootLoopMitigation(4);

        verify(rescuePartyObserver, never()).executeBootLoopMitigation(5);



        for (int i = 0; i < PackageWatchdog.DEFAULT_BOOT_LOOP_MITIGATION_INCREMENT; i++) {

        watchdog.noteBoot();

        }



        verify(rescuePartyObserver).executeBootLoopMitigation(5);

        verify(rescuePartyObserver, never()).executeBootLoopMitigation(6);



        for (int i = 0; i < PackageWatchdog.DEFAULT_BOOT_LOOP_MITIGATION_INCREMENT; i++) {

        watchdog.noteBoot();

        }



        verify(rescuePartyObserver).executeBootLoopMitigation(6);

        verify(rescuePartyObserver, never()).executeBootLoopMitigation(7);

    }
@@ -268,11 +262,11 @@ public class CrashRecoveryTest { 
                setUpRollbackPackageHealthObserver(watchdog);



        verify(rollbackObserver, never()).executeBootLoopMitigation(1);

        int bootCounter = 0;



        for (int i = 0; i < PackageWatchdog.DEFAULT_BOOT_LOOP_TRIGGER_COUNT; i++) {

            watchdog.noteBoot();

            bootCounter += 1;

        }



        verify(rollbackObserver).executeBootLoopMitigation(1);

        verify(rollbackObserver, never()).executeBootLoopMitigation(2);
@@ -280,19 +274,16 @@ public class CrashRecoveryTest { 
        when(mRollbackManager.getAvailableRollbacks()).thenReturn(List.of(ROLLBACK_INFO_HIGH,

                ROLLBACK_INFO_MANUAL));



        int bootLoopThreshold = PackageWatchdog.DEFAULT_BOOT_LOOP_THRESHOLD - bootCounter;

        for (int i = 0; i < bootLoopThreshold; i++) {

        watchdog.noteBoot();

        }



        verify(rollbackObserver).executeBootLoopMitigation(2);

        verify(rollbackObserver, never()).executeBootLoopMitigation(3);



        // Update the list of available rollbacks after executing bootloop mitigation once

        when(mRollbackManager.getAvailableRollbacks()).thenReturn(List.of(ROLLBACK_INFO_MANUAL));



        for (int i = 0; i < PackageWatchdog.DEFAULT_BOOT_LOOP_MITIGATION_INCREMENT; i++) {

        watchdog.noteBoot();

        }



        verify(rollbackObserver, never()).executeBootLoopMitigation(3);

    }
@@ -305,27 +296,21 @@ public class CrashRecoveryTest { 


        verify(rescuePartyObserver, never()).executeBootLoopMitigation(1);

        verify(rollbackObserver, never()).executeBootLoopMitigation(1);

        int bootCounter = 0;

        for (int i = 0; i < PackageWatchdog.DEFAULT_BOOT_LOOP_TRIGGER_COUNT; i++) {

            watchdog.noteBoot();

            bootCounter += 1;

        }

        verify(rescuePartyObserver).executeBootLoopMitigation(1);

        verify(rescuePartyObserver, never()).executeBootLoopMitigation(2);

        verify(rollbackObserver, never()).executeBootLoopMitigation(1);



        for (int i = 0; i < PackageWatchdog.DEFAULT_BOOT_LOOP_MITIGATION_INCREMENT; i++) {

        watchdog.noteBoot();

            bootCounter += 1;

        }



        verify(rescuePartyObserver).executeBootLoopMitigation(2);

        verify(rescuePartyObserver, never()).executeBootLoopMitigation(3);

        verify(rollbackObserver, never()).executeBootLoopMitigation(2);



        for (int i = 0; i < PackageWatchdog.DEFAULT_BOOT_LOOP_MITIGATION_INCREMENT; i++) {

        watchdog.noteBoot();

            bootCounter += 1;

        }



        verify(rescuePartyObserver, never()).executeBootLoopMitigation(3);

        verify(rollbackObserver).executeBootLoopMitigation(1);

        verify(rollbackObserver, never()).executeBootLoopMitigation(2);
@@ -333,43 +318,46 @@ public class CrashRecoveryTest { 
        when(mRollbackManager.getAvailableRollbacks()).thenReturn(List.of(ROLLBACK_INFO_HIGH,

                ROLLBACK_INFO_MANUAL));



        int bootLoopThreshold = PackageWatchdog.DEFAULT_BOOT_LOOP_THRESHOLD - bootCounter;

        for (int i = 0; i < bootLoopThreshold; i++) {

        watchdog.noteBoot();

        }



        verify(rescuePartyObserver).executeBootLoopMitigation(3);

        verify(rescuePartyObserver, never()).executeBootLoopMitigation(4);

        verify(rollbackObserver, never()).executeBootLoopMitigation(2);



        for (int i = 0; i < PackageWatchdog.DEFAULT_BOOT_LOOP_MITIGATION_INCREMENT; i++) {

        watchdog.noteBoot();

        }



        verify(rescuePartyObserver).executeBootLoopMitigation(4);

        verify(rescuePartyObserver, never()).executeBootLoopMitigation(5);

        verify(rollbackObserver, never()).executeBootLoopMitigation(2);



        for (int i = 0; i < PackageWatchdog.DEFAULT_BOOT_LOOP_MITIGATION_INCREMENT; i++) {

        watchdog.noteBoot();

        }



        verify(rescuePartyObserver).executeBootLoopMitigation(5);

        verify(rescuePartyObserver, never()).executeBootLoopMitigation(6);

        verify(rollbackObserver, never()).executeBootLoopMitigation(2);



        for (int i = 0; i < PackageWatchdog.DEFAULT_BOOT_LOOP_MITIGATION_INCREMENT; i++) {

        watchdog.noteBoot();

        }



        verify(rescuePartyObserver, never()).executeBootLoopMitigation(6);

        verify(rollbackObserver).executeBootLoopMitigation(2);

        verify(rollbackObserver, never()).executeBootLoopMitigation(3);

        // Update the list of available rollbacks after executing bootloop mitigation

        when(mRollbackManager.getAvailableRollbacks()).thenReturn(List.of(ROLLBACK_INFO_MANUAL));



        for (int i = 0; i < PackageWatchdog.DEFAULT_BOOT_LOOP_MITIGATION_INCREMENT; i++) {

        watchdog.noteBoot();

        }



        verify(rescuePartyObserver).executeBootLoopMitigation(6);

        verify(rescuePartyObserver, never()).executeBootLoopMitigation(7);

        verify(rollbackObserver, never()).executeBootLoopMitigation(3);



        moveTimeForwardAndDispatch(PackageWatchdog.DEFAULT_DEESCALATION_WINDOW_MS + 1);

        Mockito.reset(rescuePartyObserver);



        for (int i = 0; i < PackageWatchdog.DEFAULT_BOOT_LOOP_TRIGGER_COUNT; i++) {

            watchdog.noteBoot();

        }

        verify(rescuePartyObserver).executeBootLoopMitigation(1);

        verify(rescuePartyObserver, never()).executeBootLoopMitigation(2);

    }



    RollbackPackageHealthObserver setUpRollbackPackageHealthObserver(PackageWatchdog watchdog) {
@@ -506,16 +494,9 @@ public class CrashRecoveryTest { 
        }



        try {

            if (Flags.recoverabilityDetection()) {

                mSpyBootThreshold = spy(watchdog.new BootThreshold(

                    PackageWatchdog.DEFAULT_BOOT_LOOP_TRIGGER_COUNT,

                    PackageWatchdog.DEFAULT_BOOT_LOOP_TRIGGER_WINDOW_MS,

                    PackageWatchdog.DEFAULT_BOOT_LOOP_MITIGATION_INCREMENT));

            } else {

            mSpyBootThreshold = spy(watchdog.new BootThreshold(

                    PackageWatchdog.DEFAULT_BOOT_LOOP_TRIGGER_COUNT,

                    PackageWatchdog.DEFAULT_BOOT_LOOP_TRIGGER_WINDOW_MS));

            }



            doAnswer((Answer<Integer>) invocationOnMock -> {

                String storedValue = mCrashRecoveryPropertiesMap
@@ -640,5 +621,16 @@ public class CrashRecoveryTest { 
        public long uptimeMillis() {

            return mUpTimeMillis;

        }

        public void moveTimeForward(long milliSeconds) {

            mUpTimeMillis += milliSeconds;

        }

    }



    private void moveTimeForwardAndDispatch(long milliSeconds) {

        // Exhaust all due runnables now which shouldn't be executed after time-leap

        mTestLooper.dispatchAll();

        mTestClock.moveTimeForward(milliSeconds);

        mTestLooper.moveTimeForward(milliSeconds);

        mTestLooper.dispatchAll();

    }

}

tests/PackageWatchdog/src/com/android/server/PackageWatchdogTest.java

+4 −61
Original line number Diff line number Diff line
@@ -1224,7 +1224,7 @@ public class PackageWatchdogTest { 
        PackageWatchdog watchdog = createWatchdog();

        TestObserver bootObserver = new TestObserver(OBSERVER_NAME_1);

        watchdog.registerHealthObserver(bootObserver);

        for (int i = 0; i < PackageWatchdog.DEFAULT_BOOT_LOOP_THRESHOLD; i++) {

        for (int i = 0; i < 15; i++) {

            watchdog.noteBoot();

        }

        assertThat(bootObserver.mitigatedBootLoop()).isTrue();
@@ -1261,22 +1261,6 @@ public class PackageWatchdogTest { 
        assertThat(bootObserver.mitigatedBootLoop()).isFalse();

    }



    /**

     * Ensure that boot loop mitigation is not done when the number of boots does not meet the

     * threshold.

     */

    @Test

    public void testBootLoopDetection_doesNotMeetThresholdRecoverabilityHighImpact() {

        PackageWatchdog watchdog = createWatchdog();

        TestObserver bootObserver = new TestObserver(OBSERVER_NAME_1,

                PackageHealthObserverImpact.USER_IMPACT_LEVEL_80);

        watchdog.registerHealthObserver(bootObserver);

        for (int i = 0; i < PackageWatchdog.DEFAULT_BOOT_LOOP_THRESHOLD - 1; i++) {

            watchdog.noteBoot();

        }

        assertThat(bootObserver.mitigatedBootLoop()).isFalse();

    }



    /**

     * Ensure that boot loop mitigation is done for the observer with the lowest user impact

     */
@@ -1306,7 +1290,7 @@ public class PackageWatchdogTest { 
        bootObserver2.setImpact(PackageHealthObserverImpact.USER_IMPACT_LEVEL_30);

        watchdog.registerHealthObserver(bootObserver1);

        watchdog.registerHealthObserver(bootObserver2);

        for (int i = 0; i < PackageWatchdog.DEFAULT_BOOT_LOOP_THRESHOLD; i++) {

        for (int i = 0; i < 15; i++) {

            watchdog.noteBoot();

        }

        assertThat(bootObserver1.mitigatedBootLoop()).isTrue();
@@ -1349,10 +1333,8 @@ public class PackageWatchdogTest { 
            watchdog.noteBoot();

        }

        for (int i = 0; i < 4; i++) {

            for (int j = 0; j < PackageWatchdog.DEFAULT_BOOT_LOOP_MITIGATION_INCREMENT; j++) {

                watchdog.noteBoot();

        }

        }



        moveTimeForwardAndDispatch(PackageWatchdog.DEFAULT_DEESCALATION_WINDOW_MS + 1);
@@ -1360,39 +1342,8 @@ public class PackageWatchdogTest { 
            watchdog.noteBoot();

        }

        for (int i = 0; i < 4; i++) {

            for (int j = 0; j < PackageWatchdog.DEFAULT_BOOT_LOOP_MITIGATION_INCREMENT; j++) {

                watchdog.noteBoot();

        }

        }



        assertThat(bootObserver.mBootMitigationCounts).isEqualTo(List.of(1, 2, 3, 4, 1, 2, 3, 4));

    }



    @Test

    public void testMultipleBootLoopMitigationRecoverabilityHighImpact() {

        PackageWatchdog watchdog = createWatchdog();

        TestObserver bootObserver = new TestObserver(OBSERVER_NAME_1,

                PackageHealthObserverImpact.USER_IMPACT_LEVEL_80);

        watchdog.registerHealthObserver(bootObserver);

        for (int j = 0; j < PackageWatchdog.DEFAULT_BOOT_LOOP_THRESHOLD - 1; j++) {

            watchdog.noteBoot();

        }

        for (int i = 0; i < 4; i++) {

            for (int j = 0; j < PackageWatchdog.DEFAULT_BOOT_LOOP_MITIGATION_INCREMENT; j++) {

                watchdog.noteBoot();

            }

        }



        moveTimeForwardAndDispatch(PackageWatchdog.DEFAULT_DEESCALATION_WINDOW_MS + 1);



        for (int j = 0; j < PackageWatchdog.DEFAULT_BOOT_LOOP_THRESHOLD - 1; j++) {

            watchdog.noteBoot();

        }

        for (int i = 0; i < 4; i++) {

            for (int j = 0; j < PackageWatchdog.DEFAULT_BOOT_LOOP_MITIGATION_INCREMENT; j++) {

                watchdog.noteBoot();

            }

        }



        assertThat(bootObserver.mBootMitigationCounts).isEqualTo(List.of(1, 2, 3, 4, 1, 2, 3, 4));

    }
@@ -1642,8 +1593,7 @@ public class PackageWatchdogTest { 


        mSpyBootThreshold = spy(watchdog.new BootThreshold(

                PackageWatchdog.DEFAULT_BOOT_LOOP_TRIGGER_COUNT,

                PackageWatchdog.DEFAULT_BOOT_LOOP_TRIGGER_WINDOW_MS,

                PackageWatchdog.DEFAULT_BOOT_LOOP_MITIGATION_INCREMENT));

                PackageWatchdog.DEFAULT_BOOT_LOOP_TRIGGER_WINDOW_MS));



        watchdog.saveAllObserversBootMitigationCountToMetadata(filePath);
@@ -1798,16 +1748,9 @@ public class PackageWatchdogTest { 
        mCrashRecoveryPropertiesMap = new HashMap<>();



        try {

            if (Flags.recoverabilityDetection()) {

                mSpyBootThreshold = spy(watchdog.new BootThreshold(

                    PackageWatchdog.DEFAULT_BOOT_LOOP_TRIGGER_COUNT,

                    PackageWatchdog.DEFAULT_BOOT_LOOP_TRIGGER_WINDOW_MS,

                    PackageWatchdog.DEFAULT_BOOT_LOOP_MITIGATION_INCREMENT));

            } else {

            mSpyBootThreshold = spy(watchdog.new BootThreshold(

                    PackageWatchdog.DEFAULT_BOOT_LOOP_TRIGGER_COUNT,

                    PackageWatchdog.DEFAULT_BOOT_LOOP_TRIGGER_WINDOW_MS));

            }



            doAnswer((Answer<Integer>) invocationOnMock -> {

                String storedValue = mCrashRecoveryPropertiesMap