Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit cde1c58d authored by Alexei Nicoara's avatar Alexei Nicoara Committed by Cherrypicker Worker
Browse files

Adding reboot throttling 

For crashes during boot, Rescue Party will start rebooting and offering
factory resets as soon it detects those crashes. If the user
continuously rejects the factory reset prompts then the device will keep
rebooting and will never finish boot. There are some cases when the
device could have been saved if it would have finished the boot
(by server-side mainline rollbacks, GMS auto-updates, flag changes).

Adding throttling mechanism for reboots and factory resets. Rescue Party
won't be allowed to do new reboots and factory resets 10 minutes after a
factory reset was offered.

Test: atest RescuePartyTest
Bug: 264885897
Change-Id: I2dae2801b46977d25609a012270351ab0335531c
(cherry picked from https://googleplex-android-review.googlesource.com/q/commit:70e8cc604da81bfb10a16dc1b1d0e3f55ed307bb)
Merged-In: I2dae2801b46977d25609a012270351ab0335531c
parent 46e1caeb

services/core/java/com/android/server/RescueParty.java

+30 −12
Original line number Original line Diff line number Diff line
@@ -79,6 +79,7 @@ public class RescueParty { 
    static final String PROP_ATTEMPTING_FACTORY_RESET = "sys.attempting_factory_reset";

    static final String PROP_ATTEMPTING_FACTORY_RESET = "sys.attempting_factory_reset";

    static final String PROP_ATTEMPTING_REBOOT = "sys.attempting_reboot";

    static final String PROP_ATTEMPTING_REBOOT = "sys.attempting_reboot";

    static final String PROP_MAX_RESCUE_LEVEL_ATTEMPTED = "sys.max_rescue_level_attempted";

    static final String PROP_MAX_RESCUE_LEVEL_ATTEMPTED = "sys.max_rescue_level_attempted";

    static final String PROP_LAST_FACTORY_RESET_TIME_MS = "persist.sys.last_factory_reset";

    @VisibleForTesting

    @VisibleForTesting

    static final int LEVEL_NONE = 0;

    static final int LEVEL_NONE = 0;

    @VisibleForTesting

    @VisibleForTesting
@@ -105,10 +106,11 @@ public class RescueParty { 
    @VisibleForTesting

    @VisibleForTesting

    static final String NAMESPACE_TO_PACKAGE_MAPPING_FLAG =

    static final String NAMESPACE_TO_PACKAGE_MAPPING_FLAG =

            "namespace_to_package_mapping";

            "namespace_to_package_mapping";

    @VisibleForTesting

    static final long FACTORY_RESET_THROTTLE_DURATION_MS = TimeUnit.MINUTES.toMillis(10);





    private static final String NAME = "rescue-party-observer";

    private static final String NAME = "rescue-party-observer";







    private static final String PROP_DISABLE_RESCUE = "persist.sys.disable_rescue";

    private static final String PROP_DISABLE_RESCUE = "persist.sys.disable_rescue";

    private static final String PROP_VIRTUAL_DEVICE = "ro.hardware.virtual_device";

    private static final String PROP_VIRTUAL_DEVICE = "ro.hardware.virtual_device";

    private static final String PROP_DEVICE_CONFIG_DISABLE_FLAG =

    private static final String PROP_DEVICE_CONFIG_DISABLE_FLAG =
@@ -327,8 +329,8 @@ public class RescueParty { 
        }

        }

    }

    }





    private static int getMaxRescueLevel(boolean mayPerformFactoryReset) {

    private static int getMaxRescueLevel(boolean mayPerformReboot) {

        if (!mayPerformFactoryReset

        if (!mayPerformReboot

                || SystemProperties.getBoolean(PROP_DISABLE_FACTORY_RESET_FLAG, false)) {

                || SystemProperties.getBoolean(PROP_DISABLE_FACTORY_RESET_FLAG, false)) {

            return LEVEL_RESET_SETTINGS_TRUSTED_DEFAULTS;

            return LEVEL_RESET_SETTINGS_TRUSTED_DEFAULTS;

        }

        }
@@ -339,11 +341,11 @@ public class RescueParty { 
     * Get the rescue level to perform if this is the n-th attempt at mitigating failure.

     * Get the rescue level to perform if this is the n-th attempt at mitigating failure.

     *

     *

     * @param mitigationCount: the mitigation attempt number (1 = first attempt etc.)

     * @param mitigationCount: the mitigation attempt number (1 = first attempt etc.)

     * @param mayPerformFactoryReset: whether or not a factory reset may be performed for the given

     * @param mayPerformReboot: whether or not a reboot and factory reset may be performed

     *                              failure.

     *                          for the given failure.

     * @return the rescue level for the n-th mitigation attempt.

     * @return the rescue level for the n-th mitigation attempt.

     */

     */

    private static int getRescueLevel(int mitigationCount, boolean mayPerformFactoryReset) {

    private static int getRescueLevel(int mitigationCount, boolean mayPerformReboot) {

        if (mitigationCount == 1) {

        if (mitigationCount == 1) {

            return LEVEL_RESET_SETTINGS_UNTRUSTED_DEFAULTS;

            return LEVEL_RESET_SETTINGS_UNTRUSTED_DEFAULTS;

        } else if (mitigationCount == 2) {

        } else if (mitigationCount == 2) {
@@ -351,9 +353,9 @@ public class RescueParty { 
        } else if (mitigationCount == 3) {

        } else if (mitigationCount == 3) {

            return LEVEL_RESET_SETTINGS_TRUSTED_DEFAULTS;

            return LEVEL_RESET_SETTINGS_TRUSTED_DEFAULTS;

        } else if (mitigationCount == 4) {

        } else if (mitigationCount == 4) {

            return Math.min(getMaxRescueLevel(mayPerformFactoryReset), LEVEL_WARM_REBOOT);

            return Math.min(getMaxRescueLevel(mayPerformReboot), LEVEL_WARM_REBOOT);

        } else if (mitigationCount >= 5) {

        } else if (mitigationCount >= 5) {

            return Math.min(getMaxRescueLevel(mayPerformFactoryReset), LEVEL_FACTORY_RESET);

            return Math.min(getMaxRescueLevel(mayPerformReboot), LEVEL_FACTORY_RESET);

        } else {

        } else {

            Slog.w(TAG, "Expected positive mitigation count, was " + mitigationCount);

            Slog.w(TAG, "Expected positive mitigation count, was " + mitigationCount);

            return LEVEL_NONE;

            return LEVEL_NONE;
@@ -450,6 +452,8 @@ public class RescueParty { 
                    break;

                    break;

                }

                }

                SystemProperties.set(PROP_ATTEMPTING_FACTORY_RESET, "true");

                SystemProperties.set(PROP_ATTEMPTING_FACTORY_RESET, "true");

                long now = System.currentTimeMillis();

                SystemProperties.set(PROP_LAST_FACTORY_RESET_TIME_MS, Long.toString(now));

                runnable = new Runnable() {

                runnable = new Runnable() {

                    @Override

                    @Override

                    public void run() {

                    public void run() {
@@ -627,7 +631,7 @@ public class RescueParty { 
            if (!isDisabled() && (failureReason == PackageWatchdog.FAILURE_REASON_APP_CRASH

            if (!isDisabled() && (failureReason == PackageWatchdog.FAILURE_REASON_APP_CRASH

                    || failureReason == PackageWatchdog.FAILURE_REASON_APP_NOT_RESPONDING)) {

                    || failureReason == PackageWatchdog.FAILURE_REASON_APP_NOT_RESPONDING)) {

                return mapRescueLevelToUserImpact(getRescueLevel(mitigationCount,

                return mapRescueLevelToUserImpact(getRescueLevel(mitigationCount,

                        mayPerformFactoryReset(failedPackage)));

                        mayPerformReboot(failedPackage)));

            } else {

            } else {

                return PackageHealthObserverImpact.USER_IMPACT_NONE;

                return PackageHealthObserverImpact.USER_IMPACT_NONE;

            }

            }
@@ -642,7 +646,7 @@ public class RescueParty { 
            if (failureReason == PackageWatchdog.FAILURE_REASON_APP_CRASH

            if (failureReason == PackageWatchdog.FAILURE_REASON_APP_CRASH

                    || failureReason == PackageWatchdog.FAILURE_REASON_APP_NOT_RESPONDING) {

                    || failureReason == PackageWatchdog.FAILURE_REASON_APP_NOT_RESPONDING) {

                final int level = getRescueLevel(mitigationCount,

                final int level = getRescueLevel(mitigationCount,

                        mayPerformFactoryReset(failedPackage));

                        mayPerformReboot(failedPackage));

                executeRescueLevel(mContext,

                executeRescueLevel(mContext,

                        failedPackage == null ? null : failedPackage.getPackageName(), level);

                        failedPackage == null ? null : failedPackage.getPackageName(), level);

                return true;

                return true;
@@ -683,8 +687,9 @@ public class RescueParty { 
            if (isDisabled()) {

            if (isDisabled()) {

                return false;

                return false;

            }

            }

            boolean mayPerformReboot = !shouldThrottleReboot();

            executeRescueLevel(mContext, /*failedPackage=*/ null,

            executeRescueLevel(mContext, /*failedPackage=*/ null,

                    getRescueLevel(mitigationCount, true));

                    getRescueLevel(mitigationCount, mayPerformReboot));

            return true;

            return true;

        }

        }
@@ -698,14 +703,27 @@ public class RescueParty { 
         * prompting a factory reset is an acceptable mitigation strategy for the package's

         * prompting a factory reset is an acceptable mitigation strategy for the package's

         * failure, {@code false} otherwise.

         * failure, {@code false} otherwise.

         */

         */

        private boolean mayPerformFactoryReset(@Nullable VersionedPackage failingPackage) {

        private boolean mayPerformReboot(@Nullable VersionedPackage failingPackage) {

            if (failingPackage == null) {

            if (failingPackage == null) {

                return false;

                return false;

            }

            }

            if (shouldThrottleReboot())  {

                return false;

            }





            return isPersistentSystemApp(failingPackage.getPackageName());

            return isPersistentSystemApp(failingPackage.getPackageName());

        }

        }





        /**

         * Returns {@code true} if Rescue Party is allowed to attempt a reboot or factory reset.

         * Will return {@code false} if a factory reset was already offered recently.

         */

        private boolean shouldThrottleReboot() {

            Long lastResetTime = SystemProperties.getLong(PROP_LAST_FACTORY_RESET_TIME_MS, 0);

            long now = System.currentTimeMillis();

            return now < lastResetTime + FACTORY_RESET_THROTTLE_DURATION_MS;

        }



        private boolean isPersistentSystemApp(@NonNull String packageName) {

        private boolean isPersistentSystemApp(@NonNull String packageName) {

            PackageManager pm = mContext.getPackageManager();

            PackageManager pm = mContext.getPackageManager();

            try {

            try {

services/tests/mockingservicestests/src/com/android/server/RescuePartyTest.java

+51 −0
Original line number Original line Diff line number Diff line
@@ -68,6 +68,7 @@ import java.util.HashMap; 
import java.util.HashSet;

import java.util.HashSet;

import java.util.List;

import java.util.List;

import java.util.concurrent.Executor;

import java.util.concurrent.Executor;

import java.util.concurrent.TimeUnit;





/**

/**

 * Test RescueParty.

 * Test RescueParty.
@@ -94,6 +95,9 @@ public class RescuePartyTest { 
            "persist.device_config.configuration.disable_rescue_party";

            "persist.device_config.configuration.disable_rescue_party";

    private static final String PROP_DISABLE_FACTORY_RESET_FLAG =

    private static final String PROP_DISABLE_FACTORY_RESET_FLAG =

            "persist.device_config.configuration.disable_rescue_party_factory_reset";

            "persist.device_config.configuration.disable_rescue_party_factory_reset";

    private static final String PROP_LAST_FACTORY_RESET_TIME_MS = "persist.sys.last_factory_reset";



    private static final int THROTTLING_DURATION_MIN = 10;





    private MockitoSession mSession;

    private MockitoSession mSession;

    private HashMap<String, String> mSystemSettingsMap;

    private HashMap<String, String> mSystemSettingsMap;
@@ -458,6 +462,53 @@ public class RescuePartyTest { 
        assertTrue(RescueParty.isFactoryResetPropertySet());

        assertTrue(RescueParty.isFactoryResetPropertySet());

    }

    }





    @Test

    public void testThrottlingOnBootFailures() {

        SystemProperties.set(RescueParty.PROP_ATTEMPTING_REBOOT, Boolean.toString(false));

        long now = System.currentTimeMillis();

        long beforeTimeout = now - TimeUnit.MINUTES.toMillis(THROTTLING_DURATION_MIN - 1);

        SystemProperties.set(PROP_LAST_FACTORY_RESET_TIME_MS, Long.toString(beforeTimeout));

        for (int i = 1; i <= LEVEL_FACTORY_RESET; i++) {

            noteBoot(i);

        }

        assertFalse(RescueParty.isAttemptingFactoryReset());

    }



    @Test

    public void testThrottlingOnAppCrash() {

        SystemProperties.set(RescueParty.PROP_ATTEMPTING_REBOOT, Boolean.toString(false));

        long now = System.currentTimeMillis();

        long beforeTimeout = now - TimeUnit.MINUTES.toMillis(THROTTLING_DURATION_MIN - 1);

        SystemProperties.set(PROP_LAST_FACTORY_RESET_TIME_MS, Long.toString(beforeTimeout));

        for (int i = 0; i <= LEVEL_FACTORY_RESET; i++) {

            noteAppCrash(i + 1, true);

        }

        assertFalse(RescueParty.isAttemptingFactoryReset());

    }



    @Test

    public void testNotThrottlingAfterTimeoutOnBootFailures() {

        SystemProperties.set(RescueParty.PROP_ATTEMPTING_REBOOT, Boolean.toString(false));

        long now = System.currentTimeMillis();

        long afterTimeout = now - TimeUnit.MINUTES.toMillis(THROTTLING_DURATION_MIN + 1);

        SystemProperties.set(PROP_LAST_FACTORY_RESET_TIME_MS, Long.toString(afterTimeout));

        for (int i = 1; i <= LEVEL_FACTORY_RESET; i++) {

            noteBoot(i);

        }

        assertTrue(RescueParty.isAttemptingFactoryReset());

    }

    @Test

    public void testNotThrottlingAfterTimeoutOnAppCrash() {

        SystemProperties.set(RescueParty.PROP_ATTEMPTING_REBOOT, Boolean.toString(false));

        long now = System.currentTimeMillis();

        long afterTimeout = now - TimeUnit.MINUTES.toMillis(THROTTLING_DURATION_MIN + 1);

        SystemProperties.set(PROP_LAST_FACTORY_RESET_TIME_MS, Long.toString(afterTimeout));

        for (int i = 0; i <= LEVEL_FACTORY_RESET; i++) {

            noteAppCrash(i + 1, true);

        }

        assertTrue(RescueParty.isAttemptingFactoryReset());

    }



    @Test

    @Test

    public void testNativeRescuePartyResets() {

    public void testNativeRescuePartyResets() {

        doReturn(true).when(() -> SettingsToPropertiesMapper.isNativeFlagsResetPerformed());

        doReturn(true).when(() -> SettingsToPropertiesMapper.isNativeFlagsResetPerformed());