Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit bb4675df authored by Harshit Mahajan's avatar Harshit Mahajan
Browse files

Add a level to commit all available rollbacks 

There have been a few issues (like b/244419269) which led to devices
performing factory reset step, but could have been prevented
if we committed all available rollbacks. Thus adding the step to rollback all available rollbacks as preventive step to reduce the cases where device goes into Factory Reset mode.
Also adding few more impact levels in Package Watchdog to granularly control the execution of various levels by package health observers.

Test: atest RollbackPackageHealthObserver
Bug: b/264997660

Change-Id: Ifbee3a67fe69aa9c54080e498f9afa891b6be8dc
Merged-In: Ifbee3a67fe69aa9c54080e498f9afa891b6be8dc
parent 93b5d6a3

services/core/java/com/android/server/PackageWatchdog.java

+17 −13
Original line number Diff line number Diff line
@@ -426,7 +426,7 @@ public class PackageWatchdog { 
                                }

                                int impact = registeredObserver.onHealthCheckFailed(

                                        versionedPackage, failureReason, mitigationCount);

                                if (impact != PackageHealthObserverImpact.USER_IMPACT_NONE

                                if (impact != PackageHealthObserverImpact.USER_IMPACT_LEVEL_0

                                        && impact < currentObserverImpact) {

                                    currentObserverToNotify = registeredObserver;

                                    currentObserverImpact = impact;
@@ -466,7 +466,7 @@ public class PackageWatchdog { 
            if (registeredObserver != null) {

                int impact = registeredObserver.onHealthCheckFailed(

                        failingPackage, failureReason, 1);

                if (impact != PackageHealthObserverImpact.USER_IMPACT_NONE

                if (impact != PackageHealthObserverImpact.USER_IMPACT_LEVEL_0

                        && impact < currentObserverImpact) {

                    currentObserverToNotify = registeredObserver;

                    currentObserverImpact = impact;
@@ -494,7 +494,7 @@ public class PackageWatchdog { 
                    PackageHealthObserver registeredObserver = observer.registeredObserver;

                    if (registeredObserver != null) {

                        int impact = registeredObserver.onBootLoop(mitigationCount);

                        if (impact != PackageHealthObserverImpact.USER_IMPACT_NONE

                        if (impact != PackageHealthObserverImpact.USER_IMPACT_LEVEL_0

                                && impact < currentObserverImpact) {

                            currentObserverToNotify = registeredObserver;

                            currentObserverImpact = impact;
@@ -576,19 +576,23 @@ public class PackageWatchdog { 


    /** Possible severity values of the user impact of a {@link PackageHealthObserver#execute}. */

    @Retention(SOURCE)

    @IntDef(value = {PackageHealthObserverImpact.USER_IMPACT_NONE,

                     PackageHealthObserverImpact.USER_IMPACT_LOW,

                     PackageHealthObserverImpact.USER_IMPACT_MEDIUM,

                     PackageHealthObserverImpact.USER_IMPACT_HIGH})

    @IntDef(value = {PackageHealthObserverImpact.USER_IMPACT_LEVEL_0,

                     PackageHealthObserverImpact.USER_IMPACT_LEVEL_10,

                     PackageHealthObserverImpact.USER_IMPACT_LEVEL_30,

                     PackageHealthObserverImpact.USER_IMPACT_LEVEL_50,

                     PackageHealthObserverImpact.USER_IMPACT_LEVEL_70,

                     PackageHealthObserverImpact.USER_IMPACT_LEVEL_100})

    public @interface PackageHealthObserverImpact {

        /** No action to take. */

        int USER_IMPACT_NONE = 0;

        int USER_IMPACT_LEVEL_0 = 0;

        /* Action has low user impact, user of a device will barely notice. */

        int USER_IMPACT_LOW = 1;

        /* Action has medium user impact, user of a device will likely notice. */

        int USER_IMPACT_MEDIUM = 3;

        int USER_IMPACT_LEVEL_10 = 10;

        /* Actions having medium user impact, user of a device will likely notice. */

        int USER_IMPACT_LEVEL_30 = 30;

        int USER_IMPACT_LEVEL_50 = 50;

        int USER_IMPACT_LEVEL_70 = 70;

        /* Action has high user impact, a last resort, user of a device will be very frustrated. */

        int USER_IMPACT_HIGH = 5;

        int USER_IMPACT_LEVEL_100 = 100;

    }



    /** Register instances of this interface to receive notifications on package failure. */
@@ -633,7 +637,7 @@ public class PackageWatchdog { 
         *                        boot loop (including this time).

         */

        default @PackageHealthObserverImpact int onBootLoop(int mitigationCount) {

            return PackageHealthObserverImpact.USER_IMPACT_NONE;

            return PackageHealthObserverImpact.USER_IMPACT_LEVEL_0;

        }



        /**

services/core/java/com/android/server/RescueParty.java

+6 −5
Original line number Diff line number Diff line
@@ -489,13 +489,14 @@ public class RescueParty { 
        switch(rescueLevel) {

            case LEVEL_RESET_SETTINGS_UNTRUSTED_DEFAULTS:

            case LEVEL_RESET_SETTINGS_UNTRUSTED_CHANGES:

                return PackageHealthObserverImpact.USER_IMPACT_LOW;

                return PackageHealthObserverImpact.USER_IMPACT_LEVEL_10;

            case LEVEL_RESET_SETTINGS_TRUSTED_DEFAULTS:

            case LEVEL_WARM_REBOOT:

                return PackageHealthObserverImpact.USER_IMPACT_LEVEL_50;

            case LEVEL_FACTORY_RESET:

                return PackageHealthObserverImpact.USER_IMPACT_HIGH;

                return PackageHealthObserverImpact.USER_IMPACT_LEVEL_100;

            default:

                return PackageHealthObserverImpact.USER_IMPACT_NONE;

                return PackageHealthObserverImpact.USER_IMPACT_LEVEL_0;

        }

    }
@@ -633,7 +634,7 @@ public class RescueParty { 
                return mapRescueLevelToUserImpact(getRescueLevel(mitigationCount,

                        mayPerformReboot(failedPackage)));

            } else {

                return PackageHealthObserverImpact.USER_IMPACT_NONE;

                return PackageHealthObserverImpact.USER_IMPACT_LEVEL_0;

            }

        }
@@ -677,7 +678,7 @@ public class RescueParty { 
        @Override

        public int onBootLoop(int mitigationCount) {

            if (isDisabled()) {

                return PackageHealthObserverImpact.USER_IMPACT_NONE;

                return PackageHealthObserverImpact.USER_IMPACT_LEVEL_0;

            }

            return mapRescueLevelToUserImpact(getRescueLevel(mitigationCount, true));

        }

services/core/java/com/android/server/rollback/RollbackPackageHealthObserver.java

+27 −17
Original line number Diff line number Diff line
@@ -105,36 +105,46 @@ final class RollbackPackageHealthObserver implements PackageHealthObserver { 
    @Override

    public int onHealthCheckFailed(@Nullable VersionedPackage failedPackage,

            @FailureReasons int failureReason, int mitigationCount) {

        // For native crashes, we will roll back any available rollbacks

        boolean anyRollbackAvailable = !mContext.getSystemService(RollbackManager.class)

                .getAvailableRollbacks().isEmpty();

        int impact = PackageHealthObserverImpact.USER_IMPACT_LEVEL_0;



        if (failureReason == PackageWatchdog.FAILURE_REASON_NATIVE_CRASH

                && !mContext.getSystemService(RollbackManager.class)

                .getAvailableRollbacks().isEmpty()) {

            return PackageHealthObserverImpact.USER_IMPACT_MEDIUM;

        }

        if (getAvailableRollback(failedPackage) == null) {

            // Don't handle the notification, no rollbacks available for the package

            return PackageHealthObserverImpact.USER_IMPACT_NONE;

        } else {

                && anyRollbackAvailable) {

            // For native crashes, we will directly roll back any available rollbacks

            // Note: For non-native crashes the rollback-all step has higher impact

            impact = PackageHealthObserverImpact.USER_IMPACT_LEVEL_30;

        } else if (mitigationCount == 1 && getAvailableRollback(failedPackage) != null) {

            // Rollback is available, we may get a callback into #execute

            return PackageHealthObserverImpact.USER_IMPACT_MEDIUM;

            impact = PackageHealthObserverImpact.USER_IMPACT_LEVEL_30;

        } else if (mitigationCount > 1 && anyRollbackAvailable) {

            // If any rollbacks are available, we will commit them

            impact = PackageHealthObserverImpact.USER_IMPACT_LEVEL_70;

        }



        return impact;

    }



    @Override

    public boolean execute(@Nullable VersionedPackage failedPackage,

            @FailureReasons int rollbackReason, int mitigationCount) {

        if (rollbackReason == PackageWatchdog.FAILURE_REASON_NATIVE_CRASH) {

            mHandler.post(() -> rollbackAll());

            mHandler.post(this::rollbackAll);

            return true;

        }



        if (mitigationCount == 1) {

            RollbackInfo rollback = getAvailableRollback(failedPackage);

            if (rollback == null) {

                Slog.w(TAG, "Expected rollback but no valid rollback found for " + failedPackage);

                return false;

            }

            mHandler.post(() -> rollbackPackage(rollback, failedPackage, rollbackReason));

        // Assume rollback executed successfully

        } else if (mitigationCount > 1) {

            mHandler.post(this::rollbackAll);

        }



        // Assume rollbacks executed successfully

        return true;

    }

services/tests/mockingservicestests/src/com/android/server/RescuePartyTest.java

+11 −11
Original line number Diff line number Diff line
@@ -567,36 +567,36 @@ public class RescuePartyTest { 


        // Ensure that no action is taken for cases where the failure reason is unknown

        assertEquals(observer.onHealthCheckFailed(null, PackageWatchdog.FAILURE_REASON_UNKNOWN, 1),

                PackageHealthObserverImpact.USER_IMPACT_NONE);

                PackageHealthObserverImpact.USER_IMPACT_LEVEL_0);



        // Ensure the correct user impact is returned for each mitigation count.

        assertEquals(observer.onHealthCheckFailed(null,

                PackageWatchdog.FAILURE_REASON_APP_NOT_RESPONDING, 1),

                PackageHealthObserverImpact.USER_IMPACT_LOW);

                PackageHealthObserverImpact.USER_IMPACT_LEVEL_10);



        assertEquals(observer.onHealthCheckFailed(null,

                PackageWatchdog.FAILURE_REASON_APP_NOT_RESPONDING, 2),

                PackageHealthObserverImpact.USER_IMPACT_LOW);

                PackageHealthObserverImpact.USER_IMPACT_LEVEL_10);



        assertEquals(observer.onHealthCheckFailed(null,

                PackageWatchdog.FAILURE_REASON_APP_NOT_RESPONDING, 3),

                PackageHealthObserverImpact.USER_IMPACT_HIGH);

                PackageHealthObserverImpact.USER_IMPACT_LEVEL_50);



        assertEquals(observer.onHealthCheckFailed(null,

                PackageWatchdog.FAILURE_REASON_APP_NOT_RESPONDING, 4),

                PackageHealthObserverImpact.USER_IMPACT_HIGH);

                PackageHealthObserverImpact.USER_IMPACT_LEVEL_50);

    }



    @Test

    public void testBootLoopLevels() {

        RescuePartyObserver observer = RescuePartyObserver.getInstance(mMockContext);



        assertEquals(observer.onBootLoop(0), PackageHealthObserverImpact.USER_IMPACT_NONE);

        assertEquals(observer.onBootLoop(1), PackageHealthObserverImpact.USER_IMPACT_LOW);

        assertEquals(observer.onBootLoop(2), PackageHealthObserverImpact.USER_IMPACT_LOW);

        assertEquals(observer.onBootLoop(3), PackageHealthObserverImpact.USER_IMPACT_HIGH);

        assertEquals(observer.onBootLoop(4), PackageHealthObserverImpact.USER_IMPACT_HIGH);

        assertEquals(observer.onBootLoop(5), PackageHealthObserverImpact.USER_IMPACT_HIGH);

        assertEquals(observer.onBootLoop(0), PackageHealthObserverImpact.USER_IMPACT_LEVEL_0);

        assertEquals(observer.onBootLoop(1), PackageHealthObserverImpact.USER_IMPACT_LEVEL_10);

        assertEquals(observer.onBootLoop(2), PackageHealthObserverImpact.USER_IMPACT_LEVEL_10);

        assertEquals(observer.onBootLoop(3), PackageHealthObserverImpact.USER_IMPACT_LEVEL_50);

        assertEquals(observer.onBootLoop(4), PackageHealthObserverImpact.USER_IMPACT_LEVEL_50);

        assertEquals(observer.onBootLoop(5), PackageHealthObserverImpact.USER_IMPACT_LEVEL_100);

    }



    @Test

services/tests/servicestests/src/com/android/server/rollback/RollbackPackageHealthObserverTest.java→services/tests/mockingservicestests/src/com/android/server/rollback/RollbackPackageHealthObserverTest.java

+97 −6
Original line number Diff line number Diff line
@@ -16,31 +16,65 @@ 


package com.android.server.rollback;



import static com.android.dx.mockito.inline.extended.ExtendedMockito.doAnswer;



import static com.google.common.truth.Truth.assertThat;



import static org.junit.Assert.assertEquals;

import static org.mockito.Mockito.spy;

import static org.mockito.Mockito.when;



import android.content.Context;

import android.content.pm.VersionedPackage;

import android.content.rollback.PackageRollbackInfo;

import android.content.rollback.RollbackInfo;

import android.content.rollback.RollbackManager;

import android.util.Log;

import android.util.Xml;



import androidx.test.runner.AndroidJUnit4;



import com.android.dx.mockito.inline.extended.ExtendedMockito;

import com.android.server.PackageWatchdog;

import com.android.server.SystemConfig;



import org.junit.After;

import org.junit.Before;

import org.junit.Rule;

import org.junit.Test;

import org.junit.rules.TemporaryFolder;

import org.junit.runner.RunWith;

import org.mockito.Answers;

import org.mockito.Mock;

import org.mockito.MockitoSession;

import org.mockito.quality.Strictness;

import org.mockito.stubbing.Answer;

import org.xmlpull.v1.XmlPullParser;



import java.io.BufferedWriter;

import java.io.File;

import java.io.FileWriter;

import java.io.IOException;

import java.util.List;

import java.util.Scanner;





@RunWith(AndroidJUnit4.class)

public class RollbackPackageHealthObserverTest {

    @Mock

    private Context mMockContext;

    @Mock(answer = Answers.RETURNS_DEEP_STUBS)

    private PackageWatchdog mMockPackageWatchdog;

    @Mock

    RollbackManager mRollbackManager;

    @Mock

    RollbackInfo mRollbackInfo;

    @Mock

    PackageRollbackInfo mPackageRollbackInfo;



    private MockitoSession mSession;

    private static final String APP_A = "com.package.a";

    private static final long VERSION_CODE = 1L;

    private static final String LOG_TAG = "RollbackPackageHealthObserverTest";



    private SystemConfig mSysConfig;
@@ -50,6 +84,22 @@ public class RollbackPackageHealthObserverTest { 
    @Before

    public void setup() {

        mSysConfig = new SystemConfigTestClass();



        mSession = ExtendedMockito.mockitoSession()

                .initMocks(this)

                .strictness(Strictness.LENIENT)

                .spyStatic(PackageWatchdog.class)

                .startMocking();



        // Mock PackageWatchdog

        doAnswer((Answer<PackageWatchdog>) invocationOnMock -> mMockPackageWatchdog)

                .when(() -> PackageWatchdog.getInstance(mMockContext));



    }



    @After

    public void tearDown() throws Exception {

        mSession.finishMocking();

    }



    /**
@@ -61,6 +111,47 @@ public class RollbackPackageHealthObserverTest { 
        }

    }



    @Test

    public void testHealthCheckLevels() {

        RollbackPackageHealthObserver observer =

                spy(new RollbackPackageHealthObserver(mMockContext));

        VersionedPackage testFailedPackage = new VersionedPackage(APP_A, VERSION_CODE);





        when(mMockContext.getSystemService(RollbackManager.class)).thenReturn(mRollbackManager);



        // Crashes with no rollbacks available

        assertEquals(PackageWatchdog.PackageHealthObserverImpact.USER_IMPACT_LEVEL_0,

                observer.onHealthCheckFailed(null,

                        PackageWatchdog.FAILURE_REASON_NATIVE_CRASH, 1));

        assertEquals(PackageWatchdog.PackageHealthObserverImpact.USER_IMPACT_LEVEL_0,

                observer.onHealthCheckFailed(null,

                        PackageWatchdog.FAILURE_REASON_APP_CRASH, 1));



        // Make the rollbacks available

        when(mRollbackManager.getAvailableRollbacks()).thenReturn(List.of(mRollbackInfo));

        when(mRollbackInfo.getPackages()).thenReturn(List.of(mPackageRollbackInfo));

        when(mPackageRollbackInfo.getVersionRolledBackFrom()).thenReturn(testFailedPackage);



        // native crash

        assertEquals(PackageWatchdog.PackageHealthObserverImpact.USER_IMPACT_LEVEL_30,

                observer.onHealthCheckFailed(null,

                        PackageWatchdog.FAILURE_REASON_NATIVE_CRASH, 1));

        // non-native crash

        assertEquals(PackageWatchdog.PackageHealthObserverImpact.USER_IMPACT_LEVEL_30,

                observer.onHealthCheckFailed(testFailedPackage,

                        PackageWatchdog.FAILURE_REASON_APP_CRASH, 1));

        // Second non-native crash again

        assertEquals(PackageWatchdog.PackageHealthObserverImpact.USER_IMPACT_LEVEL_70,

                observer.onHealthCheckFailed(testFailedPackage,

                        PackageWatchdog.FAILURE_REASON_APP_CRASH, 2));

        // Subsequent crashes when rollbacks have completed

        when(mRollbackManager.getAvailableRollbacks()).thenReturn(List.of());

        assertEquals(PackageWatchdog.PackageHealthObserverImpact.USER_IMPACT_LEVEL_0,

                observer.onHealthCheckFailed(testFailedPackage,

                        PackageWatchdog.FAILURE_REASON_APP_CRASH, 3));

    }



    /**

     * Test that isAutomaticRollbackDenied works correctly when packages that are not

     * denied are sent.