Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 01b20657 authored by Shrinidhi Hegde's avatar Shrinidhi Hegde Committed by Automerger Merge Worker
Browse files

Merge "update package watchdog to perform mitigations after every reboot once... 

Merge "update package watchdog to perform mitigations after every reboot once threshold is reached." into main am: 02d17267 am: 9143a18b

Original change: https://android-review.googlesource.com/c/platform/frameworks/base/+/3018245



Change-Id: I279c8e463e6d7b84467c6161bac8350b184bbb0a
Signed-off-by: default avatarAutomerger Merge Worker <android-build-automerger-merge-worker@system.gserviceaccount.com>
parents 14a57c14 9143a18b

packages/CrashRecovery/services/java/com/android/server/PackageWatchdog.java

+31 −45
Original line number Diff line number Diff line
@@ -137,17 +137,6 @@ public class PackageWatchdog { 
    static final int DEFAULT_BOOT_LOOP_TRIGGER_COUNT = 5;



    static final long DEFAULT_BOOT_LOOP_TRIGGER_WINDOW_MS = TimeUnit.MINUTES.toMillis(10);

    // Boot loop at which packageWatchdog starts first mitigation

    private static final String BOOT_LOOP_THRESHOLD =

            "persist.device_config.configuration.boot_loop_threshold";

    @VisibleForTesting

    static final int DEFAULT_BOOT_LOOP_THRESHOLD = 15;

    // Once boot_loop_threshold is surpassed next mitigation would be triggered after

    // specified number of reboots.

    private static final String BOOT_LOOP_MITIGATION_INCREMENT =

            "persist.device_config.configuration..boot_loop_mitigation_increment";

    @VisibleForTesting

    static final int DEFAULT_BOOT_LOOP_MITIGATION_INCREMENT = 2;



    // Threshold level at which or above user might experience significant disruption.

    private static final String MAJOR_USER_IMPACT_LEVEL_THRESHOLD =
@@ -253,15 +242,8 @@ public class PackageWatchdog { 
        mConnectivityModuleConnector = connectivityModuleConnector;

        mSystemClock = clock;

        mNumberOfNativeCrashPollsRemaining = NUMBER_OF_NATIVE_CRASH_POLLS;

        if (Flags.recoverabilityDetection()) {

            mBootThreshold = new BootThreshold(DEFAULT_BOOT_LOOP_TRIGGER_COUNT,

                    DEFAULT_BOOT_LOOP_TRIGGER_WINDOW_MS,

                    SystemProperties.getInt(BOOT_LOOP_MITIGATION_INCREMENT,

                            DEFAULT_BOOT_LOOP_MITIGATION_INCREMENT));

        } else {

        mBootThreshold = new BootThreshold(DEFAULT_BOOT_LOOP_TRIGGER_COUNT,

                DEFAULT_BOOT_LOOP_TRIGGER_WINDOW_MS);

        }



        loadFromFile();

        sPackageWatchdog = this;
@@ -526,10 +508,16 @@ public class PackageWatchdog { 
    /**

     * Called when the system server boots. If the system server is detected to be in a boot loop,

     * query each observer and perform the mitigation action with the lowest user impact.

     *

     * Note: PackageWatchdog considers system_server restart loop as bootloop. Full reboots

     * are not counted in bootloop.

     */

    @SuppressWarnings("GuardedBy")

    public void noteBoot() {

        synchronized (mLock) {

            // if boot count has reached threshold, start mitigation.

            // We wait until threshold number of restarts only for the first time. Perform

            // mitigations for every restart after that.

            boolean mitigate = mBootThreshold.incrementAndTest();

            if (mitigate) {

                if (!Flags.recoverabilityDetection()) {
@@ -557,9 +545,6 @@ public class PackageWatchdog { 
                }

                if (currentObserverToNotify != null) {

                    if (Flags.recoverabilityDetection()) {

                        if (currentObserverImpact < getUserImpactLevelLimit()

                                || (currentObserverImpact >= getUserImpactLevelLimit()

                                        && mBootThreshold.getCount() >= getBootLoopThreshold())) {

                        int currentObserverMitigationCount =

                                currentObserverInternal.getBootMitigationCount() + 1;

                        currentObserverInternal.setBootMitigationCount(
@@ -567,7 +552,6 @@ public class PackageWatchdog { 
                        saveAllObserversBootMitigationCountToMetadata(METADATA_FILE);

                        currentObserverToNotify.executeBootLoopMitigation(

                                currentObserverMitigationCount);

                        }

                    } else {

                        mBootThreshold.setMitigationCount(mitigationCount);

                        mBootThreshold.saveMitigationCountToMetadata();
@@ -647,11 +631,6 @@ public class PackageWatchdog { 
                DEFAULT_MAJOR_USER_IMPACT_LEVEL_THRESHOLD);

    }



    private int getBootLoopThreshold() {

        return SystemProperties.getInt(BOOT_LOOP_THRESHOLD,

                DEFAULT_BOOT_LOOP_THRESHOLD);

    }



    /** Possible severity values of the user impact of a {@link PackageHealthObserver#execute}. */

    @Retention(SOURCE)

    @IntDef(value = {PackageHealthObserverImpact.USER_IMPACT_LEVEL_0,
@@ -1827,16 +1806,10 @@ public class PackageWatchdog { 


        private final int mBootTriggerCount;

        private final long mTriggerWindow;

        private final int mBootMitigationIncrement;



        BootThreshold(int bootTriggerCount, long triggerWindow) {

            this(bootTriggerCount, triggerWindow, /*bootMitigationIncrement=*/ 1);

        }



        BootThreshold(int bootTriggerCount, long triggerWindow, int bootMitigationIncrement) {

            this.mBootTriggerCount = bootTriggerCount;

            this.mTriggerWindow = triggerWindow;

            this.mBootMitigationIncrement = bootMitigationIncrement;

        }



        public void reset() {
@@ -1915,6 +1888,7 @@ public class PackageWatchdog { 
            } else {

                readMitigationCountFromMetadataIfNecessary();

            }



            final long now = mSystemClock.uptimeMillis();

            if (now - getStart() < 0) {

                Slog.e(TAG, "Window was less than zero. Resetting start to current time.");
@@ -1939,20 +1913,32 @@ public class PackageWatchdog { 
                setCount(count);

                EventLogTags.writeRescueNote(Process.ROOT_UID, count, window);

                if (Flags.recoverabilityDetection()) {

                    boolean mitigate = (count >= mBootTriggerCount)

                            && (count - mBootTriggerCount) % mBootMitigationIncrement == 0;

                    return mitigate;

                    // After a reboot (e.g. by WARM_REBOOT or mainline rollback) we apply

                    // mitigations without waiting for DEFAULT_BOOT_LOOP_TRIGGER_COUNT.

                    return (count >= mBootTriggerCount)

                            || (performedMitigationsDuringWindow() && count > 1);

                }

                return count >= mBootTriggerCount;

            }

        }



        @GuardedBy("mLock")

        private boolean performedMitigationsDuringWindow() {

            for (ObserverInternal observerInternal: mAllObservers.values()) {

                if (observerInternal.getBootMitigationCount() > 0) {

                    return true;

                }

            }

            return false;

        }



        @GuardedBy("mLock")

        private void resetAllObserversBootMitigationCount() {

            for (int i = 0; i < mAllObservers.size(); i++) {

                final ObserverInternal observer = mAllObservers.valueAt(i);

                observer.setBootMitigationCount(0);

            }

            saveAllObserversBootMitigationCountToMetadata(METADATA_FILE);

        }



        @GuardedBy("mLock")

services/tests/mockingservicestests/src/com/android/server/RescuePartyTest.java

+1 −8
Original line number Diff line number Diff line
@@ -1111,16 +1111,9 @@ public class RescuePartyTest { 


        // mock properties in BootThreshold

        try {

            if (Flags.recoverabilityDetection()) {

                mSpyBootThreshold = spy(watchdog.new BootThreshold(

                    PackageWatchdog.DEFAULT_BOOT_LOOP_TRIGGER_COUNT,

                    PackageWatchdog.DEFAULT_BOOT_LOOP_TRIGGER_WINDOW_MS,

                    PackageWatchdog.DEFAULT_BOOT_LOOP_MITIGATION_INCREMENT));

            } else {

            mSpyBootThreshold = spy(watchdog.new BootThreshold(

                    PackageWatchdog.DEFAULT_BOOT_LOOP_TRIGGER_COUNT,

                    PackageWatchdog.DEFAULT_BOOT_LOOP_TRIGGER_WINDOW_MS));

            }

            mCrashRecoveryPropertiesMap = new HashMap<>();



            doAnswer((Answer<Integer>) invocationOnMock -> {

tests/PackageWatchdog/src/com/android/server/CrashRecoveryTest.java

+54 −62
Original line number Diff line number Diff line
@@ -66,6 +66,7 @@ import org.mockito.Answers; 
import org.mockito.ArgumentCaptor;

import org.mockito.Captor;

import org.mockito.Mock;

import org.mockito.Mockito;

import org.mockito.MockitoAnnotations;

import org.mockito.MockitoSession;

import org.mockito.quality.Strictness;
@@ -220,43 +221,36 @@ public class CrashRecoveryTest { 
        RescuePartyObserver rescuePartyObserver = setUpRescuePartyObserver(watchdog);



        verify(rescuePartyObserver, never()).executeBootLoopMitigation(1);

        int bootCounter = 0;



        for (int i = 0; i < PackageWatchdog.DEFAULT_BOOT_LOOP_TRIGGER_COUNT; i++) {

            watchdog.noteBoot();

            bootCounter += 1;

        }



        verify(rescuePartyObserver).executeBootLoopMitigation(1);

        verify(rescuePartyObserver, never()).executeBootLoopMitigation(2);



        for (int i = 0; i < PackageWatchdog.DEFAULT_BOOT_LOOP_MITIGATION_INCREMENT; i++) {

        watchdog.noteBoot();

            bootCounter += 1;

        }



        verify(rescuePartyObserver).executeBootLoopMitigation(2);

        verify(rescuePartyObserver, never()).executeBootLoopMitigation(3);



        int bootLoopThreshold = PackageWatchdog.DEFAULT_BOOT_LOOP_THRESHOLD - bootCounter;

        for (int i = 0; i < bootLoopThreshold; i++) {

        watchdog.noteBoot();

        }



        verify(rescuePartyObserver).executeBootLoopMitigation(3);

        verify(rescuePartyObserver, never()).executeBootLoopMitigation(4);



        for (int i = 0; i < PackageWatchdog.DEFAULT_BOOT_LOOP_MITIGATION_INCREMENT; i++) {

        watchdog.noteBoot();

        }



        verify(rescuePartyObserver).executeBootLoopMitigation(4);

        verify(rescuePartyObserver, never()).executeBootLoopMitigation(5);



        for (int i = 0; i < PackageWatchdog.DEFAULT_BOOT_LOOP_MITIGATION_INCREMENT; i++) {

        watchdog.noteBoot();

        }



        verify(rescuePartyObserver).executeBootLoopMitigation(5);

        verify(rescuePartyObserver, never()).executeBootLoopMitigation(6);



        for (int i = 0; i < PackageWatchdog.DEFAULT_BOOT_LOOP_MITIGATION_INCREMENT; i++) {

        watchdog.noteBoot();

        }



        verify(rescuePartyObserver).executeBootLoopMitigation(6);

        verify(rescuePartyObserver, never()).executeBootLoopMitigation(7);

    }
@@ -268,11 +262,11 @@ public class CrashRecoveryTest { 
                setUpRollbackPackageHealthObserver(watchdog);



        verify(rollbackObserver, never()).executeBootLoopMitigation(1);

        int bootCounter = 0;



        for (int i = 0; i < PackageWatchdog.DEFAULT_BOOT_LOOP_TRIGGER_COUNT; i++) {

            watchdog.noteBoot();

            bootCounter += 1;

        }



        verify(rollbackObserver).executeBootLoopMitigation(1);

        verify(rollbackObserver, never()).executeBootLoopMitigation(2);
@@ -280,19 +274,16 @@ public class CrashRecoveryTest { 
        when(mRollbackManager.getAvailableRollbacks()).thenReturn(List.of(ROLLBACK_INFO_HIGH,

                ROLLBACK_INFO_MANUAL));



        int bootLoopThreshold = PackageWatchdog.DEFAULT_BOOT_LOOP_THRESHOLD - bootCounter;

        for (int i = 0; i < bootLoopThreshold; i++) {

        watchdog.noteBoot();

        }



        verify(rollbackObserver).executeBootLoopMitigation(2);

        verify(rollbackObserver, never()).executeBootLoopMitigation(3);



        // Update the list of available rollbacks after executing bootloop mitigation once

        when(mRollbackManager.getAvailableRollbacks()).thenReturn(List.of(ROLLBACK_INFO_MANUAL));



        for (int i = 0; i < PackageWatchdog.DEFAULT_BOOT_LOOP_MITIGATION_INCREMENT; i++) {

        watchdog.noteBoot();

        }



        verify(rollbackObserver, never()).executeBootLoopMitigation(3);

    }
@@ -305,27 +296,21 @@ public class CrashRecoveryTest { 


        verify(rescuePartyObserver, never()).executeBootLoopMitigation(1);

        verify(rollbackObserver, never()).executeBootLoopMitigation(1);

        int bootCounter = 0;

        for (int i = 0; i < PackageWatchdog.DEFAULT_BOOT_LOOP_TRIGGER_COUNT; i++) {

            watchdog.noteBoot();

            bootCounter += 1;

        }

        verify(rescuePartyObserver).executeBootLoopMitigation(1);

        verify(rescuePartyObserver, never()).executeBootLoopMitigation(2);

        verify(rollbackObserver, never()).executeBootLoopMitigation(1);



        for (int i = 0; i < PackageWatchdog.DEFAULT_BOOT_LOOP_MITIGATION_INCREMENT; i++) {

        watchdog.noteBoot();

            bootCounter += 1;

        }



        verify(rescuePartyObserver).executeBootLoopMitigation(2);

        verify(rescuePartyObserver, never()).executeBootLoopMitigation(3);

        verify(rollbackObserver, never()).executeBootLoopMitigation(2);



        for (int i = 0; i < PackageWatchdog.DEFAULT_BOOT_LOOP_MITIGATION_INCREMENT; i++) {

        watchdog.noteBoot();

            bootCounter += 1;

        }



        verify(rescuePartyObserver, never()).executeBootLoopMitigation(3);

        verify(rollbackObserver).executeBootLoopMitigation(1);

        verify(rollbackObserver, never()).executeBootLoopMitigation(2);
@@ -333,43 +318,46 @@ public class CrashRecoveryTest { 
        when(mRollbackManager.getAvailableRollbacks()).thenReturn(List.of(ROLLBACK_INFO_HIGH,

                ROLLBACK_INFO_MANUAL));



        int bootLoopThreshold = PackageWatchdog.DEFAULT_BOOT_LOOP_THRESHOLD - bootCounter;

        for (int i = 0; i < bootLoopThreshold; i++) {

        watchdog.noteBoot();

        }



        verify(rescuePartyObserver).executeBootLoopMitigation(3);

        verify(rescuePartyObserver, never()).executeBootLoopMitigation(4);

        verify(rollbackObserver, never()).executeBootLoopMitigation(2);



        for (int i = 0; i < PackageWatchdog.DEFAULT_BOOT_LOOP_MITIGATION_INCREMENT; i++) {

        watchdog.noteBoot();

        }



        verify(rescuePartyObserver).executeBootLoopMitigation(4);

        verify(rescuePartyObserver, never()).executeBootLoopMitigation(5);

        verify(rollbackObserver, never()).executeBootLoopMitigation(2);



        for (int i = 0; i < PackageWatchdog.DEFAULT_BOOT_LOOP_MITIGATION_INCREMENT; i++) {

        watchdog.noteBoot();

        }



        verify(rescuePartyObserver).executeBootLoopMitigation(5);

        verify(rescuePartyObserver, never()).executeBootLoopMitigation(6);

        verify(rollbackObserver, never()).executeBootLoopMitigation(2);



        for (int i = 0; i < PackageWatchdog.DEFAULT_BOOT_LOOP_MITIGATION_INCREMENT; i++) {

        watchdog.noteBoot();

        }



        verify(rescuePartyObserver, never()).executeBootLoopMitigation(6);

        verify(rollbackObserver).executeBootLoopMitigation(2);

        verify(rollbackObserver, never()).executeBootLoopMitigation(3);

        // Update the list of available rollbacks after executing bootloop mitigation

        when(mRollbackManager.getAvailableRollbacks()).thenReturn(List.of(ROLLBACK_INFO_MANUAL));



        for (int i = 0; i < PackageWatchdog.DEFAULT_BOOT_LOOP_MITIGATION_INCREMENT; i++) {

        watchdog.noteBoot();

        }



        verify(rescuePartyObserver).executeBootLoopMitigation(6);

        verify(rescuePartyObserver, never()).executeBootLoopMitigation(7);

        verify(rollbackObserver, never()).executeBootLoopMitigation(3);



        moveTimeForwardAndDispatch(PackageWatchdog.DEFAULT_DEESCALATION_WINDOW_MS + 1);

        Mockito.reset(rescuePartyObserver);



        for (int i = 0; i < PackageWatchdog.DEFAULT_BOOT_LOOP_TRIGGER_COUNT; i++) {

            watchdog.noteBoot();

        }

        verify(rescuePartyObserver).executeBootLoopMitigation(1);

        verify(rescuePartyObserver, never()).executeBootLoopMitigation(2);

    }



    RollbackPackageHealthObserver setUpRollbackPackageHealthObserver(PackageWatchdog watchdog) {
@@ -506,16 +494,9 @@ public class CrashRecoveryTest { 
        }



        try {

            if (Flags.recoverabilityDetection()) {

                mSpyBootThreshold = spy(watchdog.new BootThreshold(

                    PackageWatchdog.DEFAULT_BOOT_LOOP_TRIGGER_COUNT,

                    PackageWatchdog.DEFAULT_BOOT_LOOP_TRIGGER_WINDOW_MS,

                    PackageWatchdog.DEFAULT_BOOT_LOOP_MITIGATION_INCREMENT));

            } else {

            mSpyBootThreshold = spy(watchdog.new BootThreshold(

                    PackageWatchdog.DEFAULT_BOOT_LOOP_TRIGGER_COUNT,

                    PackageWatchdog.DEFAULT_BOOT_LOOP_TRIGGER_WINDOW_MS));

            }



            doAnswer((Answer<Integer>) invocationOnMock -> {

                String storedValue = mCrashRecoveryPropertiesMap
@@ -640,5 +621,16 @@ public class CrashRecoveryTest { 
        public long uptimeMillis() {

            return mUpTimeMillis;

        }

        public void moveTimeForward(long milliSeconds) {

            mUpTimeMillis += milliSeconds;

        }

    }



    private void moveTimeForwardAndDispatch(long milliSeconds) {

        // Exhaust all due runnables now which shouldn't be executed after time-leap

        mTestLooper.dispatchAll();

        mTestClock.moveTimeForward(milliSeconds);

        mTestLooper.moveTimeForward(milliSeconds);

        mTestLooper.dispatchAll();

    }

}

tests/PackageWatchdog/src/com/android/server/PackageWatchdogTest.java

+4 −61
Original line number Diff line number Diff line
@@ -1224,7 +1224,7 @@ public class PackageWatchdogTest { 
        PackageWatchdog watchdog = createWatchdog();

        TestObserver bootObserver = new TestObserver(OBSERVER_NAME_1);

        watchdog.registerHealthObserver(bootObserver);

        for (int i = 0; i < PackageWatchdog.DEFAULT_BOOT_LOOP_THRESHOLD; i++) {

        for (int i = 0; i < 15; i++) {

            watchdog.noteBoot();

        }

        assertThat(bootObserver.mitigatedBootLoop()).isTrue();
@@ -1261,22 +1261,6 @@ public class PackageWatchdogTest { 
        assertThat(bootObserver.mitigatedBootLoop()).isFalse();

    }



    /**

     * Ensure that boot loop mitigation is not done when the number of boots does not meet the

     * threshold.

     */

    @Test

    public void testBootLoopDetection_doesNotMeetThresholdRecoverabilityHighImpact() {

        PackageWatchdog watchdog = createWatchdog();

        TestObserver bootObserver = new TestObserver(OBSERVER_NAME_1,

                PackageHealthObserverImpact.USER_IMPACT_LEVEL_80);

        watchdog.registerHealthObserver(bootObserver);

        for (int i = 0; i < PackageWatchdog.DEFAULT_BOOT_LOOP_THRESHOLD - 1; i++) {

            watchdog.noteBoot();

        }

        assertThat(bootObserver.mitigatedBootLoop()).isFalse();

    }



    /**

     * Ensure that boot loop mitigation is done for the observer with the lowest user impact

     */
@@ -1306,7 +1290,7 @@ public class PackageWatchdogTest { 
        bootObserver2.setImpact(PackageHealthObserverImpact.USER_IMPACT_LEVEL_30);

        watchdog.registerHealthObserver(bootObserver1);

        watchdog.registerHealthObserver(bootObserver2);

        for (int i = 0; i < PackageWatchdog.DEFAULT_BOOT_LOOP_THRESHOLD; i++) {

        for (int i = 0; i < 15; i++) {

            watchdog.noteBoot();

        }

        assertThat(bootObserver1.mitigatedBootLoop()).isTrue();
@@ -1349,10 +1333,8 @@ public class PackageWatchdogTest { 
            watchdog.noteBoot();

        }

        for (int i = 0; i < 4; i++) {

            for (int j = 0; j < PackageWatchdog.DEFAULT_BOOT_LOOP_MITIGATION_INCREMENT; j++) {

                watchdog.noteBoot();

        }

        }



        moveTimeForwardAndDispatch(PackageWatchdog.DEFAULT_DEESCALATION_WINDOW_MS + 1);
@@ -1360,39 +1342,8 @@ public class PackageWatchdogTest { 
            watchdog.noteBoot();

        }

        for (int i = 0; i < 4; i++) {

            for (int j = 0; j < PackageWatchdog.DEFAULT_BOOT_LOOP_MITIGATION_INCREMENT; j++) {

                watchdog.noteBoot();

        }

        }



        assertThat(bootObserver.mBootMitigationCounts).isEqualTo(List.of(1, 2, 3, 4, 1, 2, 3, 4));

    }



    @Test

    public void testMultipleBootLoopMitigationRecoverabilityHighImpact() {

        PackageWatchdog watchdog = createWatchdog();

        TestObserver bootObserver = new TestObserver(OBSERVER_NAME_1,

                PackageHealthObserverImpact.USER_IMPACT_LEVEL_80);

        watchdog.registerHealthObserver(bootObserver);

        for (int j = 0; j < PackageWatchdog.DEFAULT_BOOT_LOOP_THRESHOLD - 1; j++) {

            watchdog.noteBoot();

        }

        for (int i = 0; i < 4; i++) {

            for (int j = 0; j < PackageWatchdog.DEFAULT_BOOT_LOOP_MITIGATION_INCREMENT; j++) {

                watchdog.noteBoot();

            }

        }



        moveTimeForwardAndDispatch(PackageWatchdog.DEFAULT_DEESCALATION_WINDOW_MS + 1);



        for (int j = 0; j < PackageWatchdog.DEFAULT_BOOT_LOOP_THRESHOLD - 1; j++) {

            watchdog.noteBoot();

        }

        for (int i = 0; i < 4; i++) {

            for (int j = 0; j < PackageWatchdog.DEFAULT_BOOT_LOOP_MITIGATION_INCREMENT; j++) {

                watchdog.noteBoot();

            }

        }



        assertThat(bootObserver.mBootMitigationCounts).isEqualTo(List.of(1, 2, 3, 4, 1, 2, 3, 4));

    }
@@ -1642,8 +1593,7 @@ public class PackageWatchdogTest { 


        mSpyBootThreshold = spy(watchdog.new BootThreshold(

                PackageWatchdog.DEFAULT_BOOT_LOOP_TRIGGER_COUNT,

                PackageWatchdog.DEFAULT_BOOT_LOOP_TRIGGER_WINDOW_MS,

                PackageWatchdog.DEFAULT_BOOT_LOOP_MITIGATION_INCREMENT));

                PackageWatchdog.DEFAULT_BOOT_LOOP_TRIGGER_WINDOW_MS));



        watchdog.saveAllObserversBootMitigationCountToMetadata(filePath);
@@ -1798,16 +1748,9 @@ public class PackageWatchdogTest { 
        mCrashRecoveryPropertiesMap = new HashMap<>();



        try {

            if (Flags.recoverabilityDetection()) {

                mSpyBootThreshold = spy(watchdog.new BootThreshold(

                    PackageWatchdog.DEFAULT_BOOT_LOOP_TRIGGER_COUNT,

                    PackageWatchdog.DEFAULT_BOOT_LOOP_TRIGGER_WINDOW_MS,

                    PackageWatchdog.DEFAULT_BOOT_LOOP_MITIGATION_INCREMENT));

            } else {

            mSpyBootThreshold = spy(watchdog.new BootThreshold(

                    PackageWatchdog.DEFAULT_BOOT_LOOP_TRIGGER_COUNT,

                    PackageWatchdog.DEFAULT_BOOT_LOOP_TRIGGER_WINDOW_MS));

            }



            doAnswer((Answer<Integer>) invocationOnMock -> {

                String storedValue = mCrashRecoveryPropertiesMap