Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 68ec39e5 authored by Gavin Corkery's avatar Gavin Corkery
Browse files

Handle mitigation count logic in RescueParty 

Rather than relying on a global rescue level property,
perform rescue logic based on the number of times
mitigation has been performed for a given package
already. This value is provided by PackageWatchdog.

Removed some logic that queried the state of the
rescue level sysprop. This property cannot be removed
yet, since the boot loop mitigation is unchanged for now.

Test: atest RescuePartyTest
Bug: 172206136
Change-Id: I90981ce202bd2649bcaf33882d9bcf7b4d1a2266
parent e1800461

services/core/java/com/android/server/RescueParty.java

+40 −23
Original line number Diff line number Diff line
@@ -77,6 +77,7 @@ public class RescueParty { 
    static final String PROP_ENABLE_RESCUE = "persist.sys.enable_rescue";

    @VisibleForTesting

    static final String PROP_RESCUE_LEVEL = "sys.rescue_level";

    static final String PROP_ATTEMPTING_FACTORY_RESET = "sys.attempting_factory_reset";

    @VisibleForTesting

    static final int LEVEL_NONE = 0;

    @VisibleForTesting
@@ -155,7 +156,7 @@ public class RescueParty { 
     * Check if we're currently attempting to reboot for a factory reset.

     */

    public static boolean isAttemptingFactoryReset() {

        return SystemProperties.getInt(PROP_RESCUE_LEVEL, LEVEL_NONE) == LEVEL_FACTORY_RESET;

        return SystemProperties.getBoolean(PROP_ATTEMPTING_FACTORY_RESET, false);

    }



    /**
@@ -230,14 +231,38 @@ public class RescueParty { 
        }

    }



    private static int getMaxRescueLevel() {

        return SystemProperties.getBoolean(PROP_DISABLE_FACTORY_RESET_FLAG, false)

                ? LEVEL_RESET_SETTINGS_TRUSTED_DEFAULTS : LEVEL_FACTORY_RESET;

    }



    /**

     * Get the rescue level to perform if this is the n-th attempt at mitigating failure.

     *

     * @param mitigationCount: the mitigation attempt number (1 = first attempt etc.)

     * @return the rescue level for the n-th mitigation attempt.

     */

    private static int getRescueLevel(int mitigationCount) {

        if (mitigationCount == 1) {

            return LEVEL_RESET_SETTINGS_UNTRUSTED_DEFAULTS;

        } else if (mitigationCount == 2) {

            return LEVEL_RESET_SETTINGS_UNTRUSTED_CHANGES;

        } else if (mitigationCount == 3) {

            return LEVEL_RESET_SETTINGS_TRUSTED_DEFAULTS;

        } else if (mitigationCount >= 4) {

            return getMaxRescueLevel();

        } else {

            Slog.w(TAG, "Expected positive mitigation count, was " + mitigationCount);

            return LEVEL_NONE;

        }

    }



    /**

     * Get the next rescue level. This indicates the next level of mitigation that may be taken.

     */

    private static int getNextRescueLevel() {

        int maxRescueLevel = SystemProperties.getBoolean(PROP_DISABLE_FACTORY_RESET_FLAG, false)

                ? LEVEL_RESET_SETTINGS_TRUSTED_DEFAULTS : LEVEL_FACTORY_RESET;

        return MathUtils.constrain(SystemProperties.getInt(PROP_RESCUE_LEVEL, LEVEL_NONE) + 1,

                LEVEL_NONE, maxRescueLevel);

                LEVEL_NONE, getMaxRescueLevel());

    }



    /**
@@ -256,7 +281,11 @@ public class RescueParty { 
    private static void executeRescueLevel(Context context, @Nullable String failedPackage) {

        final int level = SystemProperties.getInt(PROP_RESCUE_LEVEL, LEVEL_NONE);

        if (level == LEVEL_NONE) return;

        executeRescueLevel(context, failedPackage, level);

    }



    private static void executeRescueLevel(Context context, @Nullable String failedPackage,

            int level) {

        Slog.w(TAG, "Attempting rescue level " + levelToString(level));

        try {

            executeRescueLevelInternal(context, level, failedPackage);
@@ -284,6 +313,7 @@ public class RescueParty { 
            case LEVEL_FACTORY_RESET:

                // Request the reboot from a separate thread to avoid deadlock on PackageWatchdog

                // when device shutting down.

                SystemProperties.set(PROP_ATTEMPTING_FACTORY_RESET, "true");

                Runnable runnable = new Runnable() {

                    @Override

                    public void run() {
@@ -320,15 +350,6 @@ public class RescueParty { 
        }

    }



    private static int getPackageUid(Context context, String packageName) {

        try {

            return context.getPackageManager().getPackageUid(packageName, 0);

        } catch (PackageManager.NameNotFoundException e) {

            // Since UIDs are always >= 0, this value means the UID could not be determined.

            return -1;

        }

    }



    private static void resetAllSettings(Context context, int mode, @Nullable String failedPackage)

            throws Exception {

        // Try our best to reset all settings possible, and once finished
@@ -359,7 +380,7 @@ public class RescueParty { 


    private static void resetDeviceConfig(Context context, int resetMode,

            @Nullable String failedPackage) {

        if (!shouldPerformScopedResets() || failedPackage == null) {

        if (!shouldPerformScopedResets(resetMode) || failedPackage == null) {

            resetAllAffectedNamespaces(context, resetMode);

        } else {

            performScopedReset(context, resetMode, failedPackage);
@@ -384,11 +405,8 @@ public class RescueParty { 
        }

    }



    private static boolean shouldPerformScopedResets() {

        int rescueLevel = MathUtils.constrain(

                SystemProperties.getInt(PROP_RESCUE_LEVEL, LEVEL_NONE),

                LEVEL_NONE, LEVEL_FACTORY_RESET);

        return rescueLevel <= LEVEL_RESET_SETTINGS_UNTRUSTED_CHANGES;

    private static boolean shouldPerformScopedResets(int resetMode) {

        return resetMode <= Settings.RESET_MODE_UNTRUSTED_CHANGES;

    }



    private static void performScopedReset(Context context, int resetMode,
@@ -455,7 +473,7 @@ public class RescueParty { 
                @FailureReasons int failureReason, int mitigationCount) {

            if (!isDisabled() && (failureReason == PackageWatchdog.FAILURE_REASON_APP_CRASH

                    || failureReason == PackageWatchdog.FAILURE_REASON_APP_NOT_RESPONDING)) {

                return mapRescueLevelToUserImpact(getNextRescueLevel());

                return mapRescueLevelToUserImpact(getRescueLevel(mitigationCount));

            } else {

                return PackageHealthObserverImpact.USER_IMPACT_NONE;

            }
@@ -469,10 +487,9 @@ public class RescueParty { 
            }

            if (failureReason == PackageWatchdog.FAILURE_REASON_APP_CRASH

                    || failureReason == PackageWatchdog.FAILURE_REASON_APP_NOT_RESPONDING) {

                int triggerUid = getPackageUid(mContext, failedPackage.getPackageName());

                incrementRescueLevel(triggerUid);

                final int level = getRescueLevel(mitigationCount);

                executeRescueLevel(mContext,

                        failedPackage == null ? null : failedPackage.getPackageName());

                        failedPackage == null ? null : failedPackage.getPackageName(), level);

                return true;

            } else {

                return false;

services/tests/mockingservicestests/src/com/android/server/RescuePartyTest.java

+20 −67
Original line number Diff line number Diff line
@@ -226,28 +226,20 @@ public class RescuePartyTest { 


    @Test

    public void testPersistentAppCrashDetectionWithExecutionForAllRescueLevels() {

        notePersistentAppCrash();

        notePersistentAppCrash(1);



        verifySettingsResets(Settings.RESET_MODE_UNTRUSTED_DEFAULTS, /*resetNamespaces=*/ null);

        assertEquals(RescueParty.LEVEL_RESET_SETTINGS_UNTRUSTED_DEFAULTS,

                SystemProperties.getInt(RescueParty.PROP_RESCUE_LEVEL, RescueParty.LEVEL_NONE));



        notePersistentAppCrash();

        notePersistentAppCrash(2);



        verifySettingsResets(Settings.RESET_MODE_UNTRUSTED_CHANGES, /*resetNamespaces=*/ null);

        assertEquals(RescueParty.LEVEL_RESET_SETTINGS_UNTRUSTED_CHANGES,

                SystemProperties.getInt(RescueParty.PROP_RESCUE_LEVEL, RescueParty.LEVEL_NONE));



        notePersistentAppCrash();

        notePersistentAppCrash(3);



        verifySettingsResets(Settings.RESET_MODE_TRUSTED_DEFAULTS, /*resetNamespaces=*/ null);

        assertEquals(RescueParty.LEVEL_RESET_SETTINGS_TRUSTED_DEFAULTS,

                SystemProperties.getInt(RescueParty.PROP_RESCUE_LEVEL, RescueParty.LEVEL_NONE));



        notePersistentAppCrash();



        assertEquals(LEVEL_FACTORY_RESET,

                SystemProperties.getInt(RescueParty.PROP_RESCUE_LEVEL, RescueParty.LEVEL_NONE));

        notePersistentAppCrash(4);

        assertTrue(RescueParty.isAttemptingFactoryReset());

    }



    @Test
@@ -281,25 +273,19 @@ public class RescuePartyTest { 
        final String[] expectedAllResetNamespaces =

                new String[]{NAMESPACE1, NAMESPACE2, NAMESPACE3};

        observer.execute(new VersionedPackage(

                CALLING_PACKAGE1, 1), PackageWatchdog.FAILURE_REASON_APP_CRASH);

                CALLING_PACKAGE1, 1), PackageWatchdog.FAILURE_REASON_APP_CRASH, 1);

        verifySettingsResets(Settings.RESET_MODE_UNTRUSTED_DEFAULTS, expectedResetNamespaces);

        assertEquals(RescueParty.LEVEL_RESET_SETTINGS_UNTRUSTED_DEFAULTS,

                SystemProperties.getInt(RescueParty.PROP_RESCUE_LEVEL, RescueParty.LEVEL_NONE));



        observer.execute(new VersionedPackage(

                CALLING_PACKAGE1, 1), PackageWatchdog.FAILURE_REASON_APP_NOT_RESPONDING);

                CALLING_PACKAGE1, 1), PackageWatchdog.FAILURE_REASON_APP_NOT_RESPONDING, 2);

        verifySettingsResets(Settings.RESET_MODE_UNTRUSTED_CHANGES, expectedResetNamespaces);

        assertEquals(RescueParty.LEVEL_RESET_SETTINGS_UNTRUSTED_CHANGES,

                SystemProperties.getInt(RescueParty.PROP_RESCUE_LEVEL, RescueParty.LEVEL_NONE));



        observer.execute(new VersionedPackage(

                CALLING_PACKAGE1, 1), PackageWatchdog.FAILURE_REASON_APP_NOT_RESPONDING);

                CALLING_PACKAGE1, 1), PackageWatchdog.FAILURE_REASON_APP_NOT_RESPONDING, 3);

        verifySettingsResets(Settings.RESET_MODE_TRUSTED_DEFAULTS, expectedAllResetNamespaces);

        assertEquals(RescueParty.LEVEL_RESET_SETTINGS_TRUSTED_DEFAULTS,

                SystemProperties.getInt(RescueParty.PROP_RESCUE_LEVEL, RescueParty.LEVEL_NONE));



        observer.execute(new VersionedPackage(

                CALLING_PACKAGE1, 1), PackageWatchdog.FAILURE_REASON_APP_CRASH);

                CALLING_PACKAGE1, 1), PackageWatchdog.FAILURE_REASON_APP_NOT_RESPONDING, 4);

        assertTrue(RescueParty.isAttemptingFactoryReset());

    }
@@ -341,11 +327,11 @@ public class RescuePartyTest { 
        SystemProperties.set(RescueParty.PROP_ENABLE_RESCUE, Boolean.toString(false));

        SystemProperties.set(PROP_DISABLE_RESCUE, Boolean.toString(true));

        assertEquals(RescuePartyObserver.getInstance(mMockContext).execute(sFailingPackage,

                PackageWatchdog.FAILURE_REASON_APP_NOT_RESPONDING), false);

                PackageWatchdog.FAILURE_REASON_APP_NOT_RESPONDING, 1), false);



        SystemProperties.set(RescueParty.PROP_ENABLE_RESCUE, Boolean.toString(true));

        assertTrue(RescuePartyObserver.getInstance(mMockContext).execute(sFailingPackage,

                PackageWatchdog.FAILURE_REASON_APP_NOT_RESPONDING));

                PackageWatchdog.FAILURE_REASON_APP_NOT_RESPONDING, 1));

    }



    @Test
@@ -354,7 +340,7 @@ public class RescuePartyTest { 
        SystemProperties.set(PROP_DEVICE_CONFIG_DISABLE_FLAG, Boolean.toString(true));



        assertEquals(RescuePartyObserver.getInstance(mMockContext).execute(sFailingPackage,

                PackageWatchdog.FAILURE_REASON_APP_NOT_RESPONDING), false);

                PackageWatchdog.FAILURE_REASON_APP_NOT_RESPONDING, 1), false);



        // Restore the property value initialized in SetUp()

        SystemProperties.set(RescueParty.PROP_ENABLE_RESCUE, Boolean.toString(true));
@@ -379,46 +365,24 @@ public class RescuePartyTest { 
        RescuePartyObserver observer = RescuePartyObserver.getInstance(mMockContext);



        // Ensure that no action is taken for cases where the failure reason is unknown

        SystemProperties.set(RescueParty.PROP_RESCUE_LEVEL, Integer.toString(

                LEVEL_FACTORY_RESET));

        assertEquals(observer.onHealthCheckFailed(null, PackageWatchdog.FAILURE_REASON_UNKNOWN),

        assertEquals(observer.onHealthCheckFailed(null, PackageWatchdog.FAILURE_REASON_UNKNOWN, 1),

                PackageHealthObserverImpact.USER_IMPACT_NONE);



        /*

        For the following cases, ensure that the returned user impact corresponds with the user

        impact of the next available rescue level, not the current one.

         */

        SystemProperties.set(RescueParty.PROP_RESCUE_LEVEL, Integer.toString(

                RescueParty.LEVEL_NONE));

        // Ensure the correct user impact is returned for each mitigation count.

        assertEquals(observer.onHealthCheckFailed(null,

                PackageWatchdog.FAILURE_REASON_APP_NOT_RESPONDING),

                PackageWatchdog.FAILURE_REASON_APP_NOT_RESPONDING, 1),

                PackageHealthObserverImpact.USER_IMPACT_LOW);



        SystemProperties.set(RescueParty.PROP_RESCUE_LEVEL, Integer.toString(

                RescueParty.LEVEL_RESET_SETTINGS_UNTRUSTED_DEFAULTS));

        assertEquals(observer.onHealthCheckFailed(null,

                PackageWatchdog.FAILURE_REASON_APP_NOT_RESPONDING),

                PackageWatchdog.FAILURE_REASON_APP_NOT_RESPONDING, 2),

                PackageHealthObserverImpact.USER_IMPACT_LOW);





        SystemProperties.set(RescueParty.PROP_RESCUE_LEVEL, Integer.toString(

                RescueParty.LEVEL_RESET_SETTINGS_UNTRUSTED_CHANGES));

        assertEquals(observer.onHealthCheckFailed(null,

                PackageWatchdog.FAILURE_REASON_APP_NOT_RESPONDING),

                PackageWatchdog.FAILURE_REASON_APP_NOT_RESPONDING, 3),

                PackageHealthObserverImpact.USER_IMPACT_HIGH);





        SystemProperties.set(RescueParty.PROP_RESCUE_LEVEL, Integer.toString(

                RescueParty.LEVEL_RESET_SETTINGS_TRUSTED_DEFAULTS));

        assertEquals(observer.onHealthCheckFailed(null,

                PackageWatchdog.FAILURE_REASON_APP_NOT_RESPONDING),

                PackageHealthObserverImpact.USER_IMPACT_HIGH);





        SystemProperties.set(RescueParty.PROP_RESCUE_LEVEL, Integer.toString(

                LEVEL_FACTORY_RESET));

        assertEquals(observer.onHealthCheckFailed(null,

                PackageWatchdog.FAILURE_REASON_APP_NOT_RESPONDING),

                PackageWatchdog.FAILURE_REASON_APP_NOT_RESPONDING, 4),

                PackageHealthObserverImpact.USER_IMPACT_HIGH);

    }
@@ -451,17 +415,6 @@ public class RescuePartyTest { 
        assertEquals(observer.onBootLoop(), PackageHealthObserverImpact.USER_IMPACT_HIGH);

    }



    @Test

    public void testRescueLevelIncrementsWhenExecuted() {

        RescuePartyObserver observer = RescuePartyObserver.getInstance(mMockContext);

        SystemProperties.set(RescueParty.PROP_RESCUE_LEVEL, Integer.toString(

                RescueParty.LEVEL_NONE));

        observer.execute(sFailingPackage,

                PackageWatchdog.FAILURE_REASON_APP_CRASH);

        assertEquals(SystemProperties.getInt(RescueParty.PROP_RESCUE_LEVEL, -1),

                RescueParty.LEVEL_RESET_SETTINGS_UNTRUSTED_DEFAULTS);

    }



    private void verifySettingsResets(int resetMode, String[] resetNamespaces) {

        verify(() -> Settings.Global.resetToDefaultsAsUser(mMockContentResolver, null,

                resetMode, UserHandle.USER_SYSTEM));
@@ -481,9 +434,9 @@ public class RescuePartyTest { 
        RescuePartyObserver.getInstance(mMockContext).executeBootLoopMitigation();

    }



    private void notePersistentAppCrash() {

    private void notePersistentAppCrash(int mitigationCount) {

        RescuePartyObserver.getInstance(mMockContext).execute(new VersionedPackage(

                "com.package.name", 1), PackageWatchdog.FAILURE_REASON_APP_CRASH);

                "com.package.name", 1), PackageWatchdog.FAILURE_REASON_APP_CRASH, mitigationCount);

    }



    private Bundle getConfigAccessBundle(String callingPackage, String namespace) {