Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit b070225f authored by JW Wang's avatar JW Wang
Browse files

Implement two-phase rollback (4/n) 

Reduce the chance of rolling back Mainline udpates which might
contain critical security fixes.

It works as:
1. roll back only rebootless apexes if they are available
2. if native crash stops, we are good again
3. if not, roll back all remaining rollbacks
   which is the best we can do

Bug: 195517333
Test: atest StagedRollbackTest
Change-Id: If938d7f9f17957ab30d71d2cc87a148c07dd3da7
parent 858845f7

services/core/java/com/android/server/rollback/RollbackManagerServiceImpl.java

+1 −0
Original line number Diff line number Diff line
@@ -1162,6 +1162,7 @@ class RollbackManagerServiceImpl extends IRollbackManager.Stub implements Rollba 
        assertInWorkerThread();

        Slog.i(TAG, "makeRollbackAvailable id=" + rollback.info.getRollbackId());

        rollback.makeAvailable();

        mPackageHealthObserver.notifyRollbackAvailable(rollback.info);



        // TODO(zezeozue): Provide API to explicitly start observing instead

        // of doing this for all rollbacks. If we do this for all rollbacks,

services/core/java/com/android/server/rollback/RollbackPackageHealthObserver.java

+90 −1
Original line number Diff line number Diff line
@@ -51,6 +51,7 @@ import com.android.server.pm.ApexManager; 


import java.io.BufferedReader;

import java.io.File;

import java.io.FileInputStream;

import java.io.FileOutputStream;

import java.io.FileReader;

import java.io.IOException;
@@ -75,8 +76,11 @@ final class RollbackPackageHealthObserver implements PackageHealthObserver { 
    private final Handler mHandler;

    private final ApexManager mApexManager;

    private final File mLastStagedRollbackIdsFile;

    private final File mTwoPhaseRollbackEnabledFile;

    // Staged rollback ids that have been committed but their session is not yet ready

    private final Set<Integer> mPendingStagedRollbackIds = new ArraySet<>();

    // True if needing to roll back only rebootless apexes when native crash happens

    private boolean mTwoPhaseRollbackEnabled;



    RollbackPackageHealthObserver(Context context) {

        mContext = context;
@@ -86,8 +90,19 @@ final class RollbackPackageHealthObserver implements PackageHealthObserver { 
        File dataDir = new File(Environment.getDataDirectory(), "rollback-observer");

        dataDir.mkdirs();

        mLastStagedRollbackIdsFile = new File(dataDir, "last-staged-rollback-ids");

        mTwoPhaseRollbackEnabledFile = new File(dataDir, "two-phase-rollback-enabled");

        PackageWatchdog.getInstance(mContext).registerHealthObserver(this);

        mApexManager = ApexManager.getInstance();



        if (SystemProperties.getBoolean("sys.boot_completed", false)) {

            // Load the value from the file if system server has crashed and restarted

            mTwoPhaseRollbackEnabled = readBoolean(mTwoPhaseRollbackEnabledFile);

        } else {

            // Disable two-phase rollback for a normal reboot. We assume the rebootless apex

            // installed before reboot is stable if native crash didn't happen.

            mTwoPhaseRollbackEnabled = false;

            writeBoolean(mTwoPhaseRollbackEnabledFile, false);

        }

    }



    @Override
@@ -144,6 +159,31 @@ final class RollbackPackageHealthObserver implements PackageHealthObserver { 
        PackageWatchdog.getInstance(mContext).startObservingHealth(this, packages, durationMs);

    }



    @AnyThread

    void notifyRollbackAvailable(RollbackInfo rollback) {

        mHandler.post(() -> {

            // Enable two-phase rollback when a rebootless apex rollback is made available.

            // We assume the rebootless apex is stable and is less likely to be the cause

            // if native crash doesn't happen before reboot. So we will clear the flag and disable

            // two-phase rollback after reboot.

            if (isRebootlessApex(rollback)) {

                mTwoPhaseRollbackEnabled = true;

                writeBoolean(mTwoPhaseRollbackEnabledFile, true);

            }

        });

    }



    private static boolean isRebootlessApex(RollbackInfo rollback) {

        if (!rollback.isStaged()) {

            for (PackageRollbackInfo info : rollback.getPackages()) {

                if (info.isApex()) {

                    return true;

                }

            }

        }

        return false;

    }



    /** Verifies the rollback state after a reboot and schedules polling for sometime after reboot

     * to check for native crashes and mitigate them if needed.

     */
@@ -155,6 +195,7 @@ final class RollbackPackageHealthObserver implements PackageHealthObserver { 
    @WorkerThread

    private void onBootCompleted() {

        assertInWorkerThread();



        RollbackManager rollbackManager = mContext.getSystemService(RollbackManager.class);

        if (!rollbackManager.getAvailableRollbacks().isEmpty()) {

            // TODO(gavincorkery): Call into Package Watchdog from outside the observer
@@ -277,6 +318,23 @@ final class RollbackPackageHealthObserver implements PackageHealthObserver { 
        return mPendingStagedRollbackIds.isEmpty();

    }



    private static boolean readBoolean(File file) {

        try (FileInputStream fis = new FileInputStream(file)) {

            return fis.read() == 1;

        } catch (IOException ignore) {

            return false;

        }

    }



    private static void writeBoolean(File file, boolean value) {

        try (FileOutputStream fos = new FileOutputStream(file)) {

            fos.write(value ? 1 : 0);

            fos.flush();

            FileUtils.sync(fos);

        } catch (IOException ignore) {

        }

    }



    @WorkerThread

    private void saveStagedRollbackId(int stagedRollbackId, @Nullable VersionedPackage logPackage) {

        assertInWorkerThread();
@@ -420,13 +478,44 @@ final class RollbackPackageHealthObserver implements PackageHealthObserver { 
                Collections.singletonList(failedPackage), rollbackReceiver.getIntentSender());

    }



    /**

     * Two-phase rollback:

     * 1. roll back rebootless apexes first

     * 2. roll back all remaining rollbacks if native crash doesn't stop after (1) is done

     *

     * This approach gives us a better chance to correctly attribute native crash to rebootless

     * apex update without rolling back Mainline updates which might contains critical security

     * fixes.

     */

    @WorkerThread

    private boolean useTwoPhaseRollback(List<RollbackInfo> rollbacks) {

        assertInWorkerThread();

        if (!mTwoPhaseRollbackEnabled) {

            return false;

        }



        Slog.i(TAG, "Rolling back all rebootless APEX rollbacks");

        boolean found = false;

        for (RollbackInfo rollback : rollbacks) {

            if (isRebootlessApex(rollback)) {

                VersionedPackage sample = rollback.getPackages().get(0).getVersionRolledBackFrom();

                rollbackPackage(rollback, sample, PackageWatchdog.FAILURE_REASON_NATIVE_CRASH);

                found = true;

            }

        }

        return found;

    }



    @WorkerThread

    private void rollbackAll() {

        assertInWorkerThread();

        Slog.i(TAG, "Rolling back all available rollbacks");

        RollbackManager rollbackManager = mContext.getSystemService(RollbackManager.class);

        List<RollbackInfo> rollbacks = rollbackManager.getAvailableRollbacks();

        if (useTwoPhaseRollback(rollbacks)) {

            return;

        }



        Slog.i(TAG, "Rolling back all available rollbacks");

        // Add all rollback ids to mPendingStagedRollbackIds, so that we do not reboot before all

        // pending staged rollbacks are handled.

        for (RollbackInfo rollback : rollbacks) {

tests/RollbackTest/RollbackTest/src/com/android/tests/rollback/StagedRollbackTest.java

+24 −1
Original line number Diff line number Diff line
@@ -58,6 +58,7 @@ import java.util.concurrent.TimeUnit; 
public class StagedRollbackTest {

    private static final String PROPERTY_WATCHDOG_TRIGGER_FAILURE_COUNT =

            "watchdog_trigger_failure_count";

    private static final String REBOOTLESS_APEX_NAME = "test.apex.rebootless";



    /**

     * Adopts common shell permissions needed for rollback tests.
@@ -242,7 +243,7 @@ public class StagedRollbackTest { 


    @Test

    public void testRollbackRebootlessApex() throws Exception {

        final String packageName = "test.apex.rebootless";

        final String packageName = REBOOTLESS_APEX_NAME;

        assertThat(InstallUtils.getInstalledVersion(packageName)).isEqualTo(1);



        // install
@@ -267,6 +268,28 @@ public class StagedRollbackTest { 
        assertThat(InstallUtils.getInstalledVersion(packageName)).isEqualTo(1);

    }



    @Test

    public void testNativeWatchdogTriggersRebootlessApexRollback_Phase1_Install() throws Exception {

        assertThat(InstallUtils.getInstalledVersion(REBOOTLESS_APEX_NAME)).isEqualTo(1);



        TestApp apex2 = new TestApp("TestRebootlessApexV2", REBOOTLESS_APEX_NAME, 2,

                /* isApex= */ true, "test.rebootless_apex_v2.apex");

        Install.single(apex2).setEnableRollback(PackageManager.ROLLBACK_DATA_POLICY_RETAIN)

                .commit();

        Install.single(TestApp.A1).commit();

        Install.single(TestApp.A2).setEnableRollback().commit();



        RollbackUtils.waitForAvailableRollback(TestApp.A);

        RollbackUtils.waitForAvailableRollback(REBOOTLESS_APEX_NAME);

    }



    @Test

    public void testNativeWatchdogTriggersRebootlessApexRollback_Phase2_Verify() throws Exception {

        // Check only rebootless apex is rolled back. Other rollbacks should remain unchanged.

        assertThat(RollbackUtils.getCommittedRollback(REBOOTLESS_APEX_NAME)).isNotNull();

        assertThat(RollbackUtils.getAvailableRollback(TestApp.A)).isNotNull();

    }



    @Test

    public void hasMainlineModule() throws Exception {

        String pkgName = getModuleMetadataPackageName();

tests/RollbackTest/StagedRollbackTest/src/com/android/tests/rollback/host/StagedRollbackTest.java

+26 −0
Original line number Diff line number Diff line
@@ -191,6 +191,18 @@ public class StagedRollbackTest extends BaseHostJUnit4Test { 
        runPhase("testRollbackRebootlessApex");

    }



    /**

     * Tests only rebootless apex (if any) is rolled back when native crash happens

     */

    @Test

    public void testNativeWatchdogTriggersRebootlessApexRollback() throws Exception {

        pushTestApex("test.rebootless_apex_v1.apex");

        runPhase("testNativeWatchdogTriggersRebootlessApexRollback_Phase1_Install");

        crashProcess("system_server", NATIVE_CRASHES_THRESHOLD);

        getDevice().waitForDeviceAvailable();

        runPhase("testNativeWatchdogTriggersRebootlessApexRollback_Phase2_Verify");

    }



    /**

     * Tests that packages are monitored across multiple reboots.

     */
@@ -253,4 +265,18 @@ public class StagedRollbackTest extends BaseHostJUnit4Test { 
            return false;

        }

    }



    private void crashProcess(String processName, int numberOfCrashes) throws Exception {

        String pid = "";

        String lastPid = "invalid";

        for (int i = 0; i < numberOfCrashes; ++i) {

            // This condition makes sure before we kill the process, the process is running AND

            // the last crash was finished.

            while ("".equals(pid) || lastPid.equals(pid)) {

                pid = getDevice().executeShellCommand("pidof " + processName);

            }

            getDevice().executeShellCommand("kill " + pid);

            lastPid = pid;

        }

    }

}