Merge "Implement two-phase rollback (4/n)" (2ea1d2d1) · Commits · e / os / android_frameworks_base

services/core/java/com/android/server/rollback/RollbackManagerServiceImpl.java

+1 −0

Original line number	Diff line number	Diff line
		@@ -1162,6 +1162,7 @@ class RollbackManagerServiceImpl extends IRollbackManager.Stub implements Rollba
		assertInWorkerThread();
		Slog.i(TAG, "makeRollbackAvailable id=" + rollback.info.getRollbackId());
		rollback.makeAvailable();
		mPackageHealthObserver.notifyRollbackAvailable(rollback.info);

		// TODO(zezeozue): Provide API to explicitly start observing instead
		// of doing this for all rollbacks. If we do this for all rollbacks,

services/core/java/com/android/server/rollback/RollbackPackageHealthObserver.java

+90 −1

Original line number	Diff line number	Diff line
		@@ -51,6 +51,7 @@ import com.android.server.pm.ApexManager;

		import java.io.BufferedReader;
		import java.io.File;
		import java.io.FileInputStream;
		import java.io.FileOutputStream;
		import java.io.FileReader;
		import java.io.IOException;
		@@ -75,8 +76,11 @@ final class RollbackPackageHealthObserver implements PackageHealthObserver {
		private final Handler mHandler;
		private final ApexManager mApexManager;
		private final File mLastStagedRollbackIdsFile;
		private final File mTwoPhaseRollbackEnabledFile;
		// Staged rollback ids that have been committed but their session is not yet ready
		private final Set<Integer> mPendingStagedRollbackIds = new ArraySet<>();
		// True if needing to roll back only rebootless apexes when native crash happens
		private boolean mTwoPhaseRollbackEnabled;

		RollbackPackageHealthObserver(Context context) {
		mContext = context;
		@@ -86,8 +90,19 @@ final class RollbackPackageHealthObserver implements PackageHealthObserver {
		File dataDir = new File(Environment.getDataDirectory(), "rollback-observer");
		dataDir.mkdirs();
		mLastStagedRollbackIdsFile = new File(dataDir, "last-staged-rollback-ids");
		mTwoPhaseRollbackEnabledFile = new File(dataDir, "two-phase-rollback-enabled");
		PackageWatchdog.getInstance(mContext).registerHealthObserver(this);
		mApexManager = ApexManager.getInstance();

		if (SystemProperties.getBoolean("sys.boot_completed", false)) {
		// Load the value from the file if system server has crashed and restarted
		mTwoPhaseRollbackEnabled = readBoolean(mTwoPhaseRollbackEnabledFile);
		} else {
		// Disable two-phase rollback for a normal reboot. We assume the rebootless apex
		// installed before reboot is stable if native crash didn't happen.
		mTwoPhaseRollbackEnabled = false;
		writeBoolean(mTwoPhaseRollbackEnabledFile, false);
		}
		}

		@Override
		@@ -144,6 +159,31 @@ final class RollbackPackageHealthObserver implements PackageHealthObserver {
		PackageWatchdog.getInstance(mContext).startObservingHealth(this, packages, durationMs);
		}

		@AnyThread
		void notifyRollbackAvailable(RollbackInfo rollback) {
		mHandler.post(() -> {
		// Enable two-phase rollback when a rebootless apex rollback is made available.
		// We assume the rebootless apex is stable and is less likely to be the cause
		// if native crash doesn't happen before reboot. So we will clear the flag and disable
		// two-phase rollback after reboot.
		if (isRebootlessApex(rollback)) {
		mTwoPhaseRollbackEnabled = true;
		writeBoolean(mTwoPhaseRollbackEnabledFile, true);
		}
		});
		}

		private static boolean isRebootlessApex(RollbackInfo rollback) {
		if (!rollback.isStaged()) {
		for (PackageRollbackInfo info : rollback.getPackages()) {
		if (info.isApex()) {
		return true;
		}
		}
		}
		return false;
		}

		/** Verifies the rollback state after a reboot and schedules polling for sometime after reboot
		* to check for native crashes and mitigate them if needed.
		*/
		@@ -155,6 +195,7 @@ final class RollbackPackageHealthObserver implements PackageHealthObserver {
		@WorkerThread
		private void onBootCompleted() {
		assertInWorkerThread();

		RollbackManager rollbackManager = mContext.getSystemService(RollbackManager.class);
		if (!rollbackManager.getAvailableRollbacks().isEmpty()) {
		// TODO(gavincorkery): Call into Package Watchdog from outside the observer
		@@ -277,6 +318,23 @@ final class RollbackPackageHealthObserver implements PackageHealthObserver {
		return mPendingStagedRollbackIds.isEmpty();
		}

		private static boolean readBoolean(File file) {
		try (FileInputStream fis = new FileInputStream(file)) {
		return fis.read() == 1;
		} catch (IOException ignore) {
		return false;
		}
		}

		private static void writeBoolean(File file, boolean value) {
		try (FileOutputStream fos = new FileOutputStream(file)) {
		fos.write(value ? 1 : 0);
		fos.flush();
		FileUtils.sync(fos);
		} catch (IOException ignore) {
		}
		}

		@WorkerThread
		private void saveStagedRollbackId(int stagedRollbackId, @Nullable VersionedPackage logPackage) {
		assertInWorkerThread();
		@@ -420,13 +478,44 @@ final class RollbackPackageHealthObserver implements PackageHealthObserver {
		Collections.singletonList(failedPackage), rollbackReceiver.getIntentSender());
		}

		/**
		* Two-phase rollback:
		* 1. roll back rebootless apexes first
		* 2. roll back all remaining rollbacks if native crash doesn't stop after (1) is done
		*
		* This approach gives us a better chance to correctly attribute native crash to rebootless
		* apex update without rolling back Mainline updates which might contains critical security
		* fixes.
		*/
		@WorkerThread
		private boolean useTwoPhaseRollback(List<RollbackInfo> rollbacks) {
		assertInWorkerThread();
		if (!mTwoPhaseRollbackEnabled) {
		return false;
		}

		Slog.i(TAG, "Rolling back all rebootless APEX rollbacks");
		boolean found = false;
		for (RollbackInfo rollback : rollbacks) {
		if (isRebootlessApex(rollback)) {
		VersionedPackage sample = rollback.getPackages().get(0).getVersionRolledBackFrom();
		rollbackPackage(rollback, sample, PackageWatchdog.FAILURE_REASON_NATIVE_CRASH);
		found = true;
		}
		}
		return found;
		}

		@WorkerThread
		private void rollbackAll() {
		assertInWorkerThread();
		Slog.i(TAG, "Rolling back all available rollbacks");
		RollbackManager rollbackManager = mContext.getSystemService(RollbackManager.class);
		List<RollbackInfo> rollbacks = rollbackManager.getAvailableRollbacks();
		if (useTwoPhaseRollback(rollbacks)) {
		return;
		}

		Slog.i(TAG, "Rolling back all available rollbacks");
		// Add all rollback ids to mPendingStagedRollbackIds, so that we do not reboot before all
		// pending staged rollbacks are handled.
		for (RollbackInfo rollback : rollbacks) {

tests/RollbackTest/RollbackTest/src/com/android/tests/rollback/StagedRollbackTest.java

+24 −1

Original line number	Diff line number	Diff line
		@@ -58,6 +58,7 @@ import java.util.concurrent.TimeUnit;
		public class StagedRollbackTest {
		private static final String PROPERTY_WATCHDOG_TRIGGER_FAILURE_COUNT =
		"watchdog_trigger_failure_count";
		private static final String REBOOTLESS_APEX_NAME = "test.apex.rebootless";

		/**
		* Adopts common shell permissions needed for rollback tests.
		@@ -242,7 +243,7 @@ public class StagedRollbackTest {

		@Test
		public void testRollbackRebootlessApex() throws Exception {
		final String packageName = "test.apex.rebootless";
		final String packageName = REBOOTLESS_APEX_NAME;
		assertThat(InstallUtils.getInstalledVersion(packageName)).isEqualTo(1);

		// install
		@@ -267,6 +268,28 @@ public class StagedRollbackTest {
		assertThat(InstallUtils.getInstalledVersion(packageName)).isEqualTo(1);
		}

		@Test
		public void testNativeWatchdogTriggersRebootlessApexRollback_Phase1_Install() throws Exception {
		assertThat(InstallUtils.getInstalledVersion(REBOOTLESS_APEX_NAME)).isEqualTo(1);

		TestApp apex2 = new TestApp("TestRebootlessApexV2", REBOOTLESS_APEX_NAME, 2,
		/* isApex= */ true, "test.rebootless_apex_v2.apex");
		Install.single(apex2).setEnableRollback(PackageManager.ROLLBACK_DATA_POLICY_RETAIN)
		.commit();
		Install.single(TestApp.A1).commit();
		Install.single(TestApp.A2).setEnableRollback().commit();

		RollbackUtils.waitForAvailableRollback(TestApp.A);
		RollbackUtils.waitForAvailableRollback(REBOOTLESS_APEX_NAME);
		}

		@Test
		public void testNativeWatchdogTriggersRebootlessApexRollback_Phase2_Verify() throws Exception {
		// Check only rebootless apex is rolled back. Other rollbacks should remain unchanged.
		assertThat(RollbackUtils.getCommittedRollback(REBOOTLESS_APEX_NAME)).isNotNull();
		assertThat(RollbackUtils.getAvailableRollback(TestApp.A)).isNotNull();
		}

		@Test
		public void hasMainlineModule() throws Exception {
		String pkgName = getModuleMetadataPackageName();

tests/RollbackTest/StagedRollbackTest/src/com/android/tests/rollback/host/StagedRollbackTest.java

+26 −0

Original line number	Diff line number	Diff line
		@@ -191,6 +191,18 @@ public class StagedRollbackTest extends BaseHostJUnit4Test {
		runPhase("testRollbackRebootlessApex");
		}

		/**
		* Tests only rebootless apex (if any) is rolled back when native crash happens
		*/
		@Test
		public void testNativeWatchdogTriggersRebootlessApexRollback() throws Exception {
		pushTestApex("test.rebootless_apex_v1.apex");
		runPhase("testNativeWatchdogTriggersRebootlessApexRollback_Phase1_Install");
		crashProcess("system_server", NATIVE_CRASHES_THRESHOLD);
		getDevice().waitForDeviceAvailable();
		runPhase("testNativeWatchdogTriggersRebootlessApexRollback_Phase2_Verify");
		}

		/**
		* Tests that packages are monitored across multiple reboots.
		*/
		@@ -253,4 +265,18 @@ public class StagedRollbackTest extends BaseHostJUnit4Test {
		return false;
		}
		}

		private void crashProcess(String processName, int numberOfCrashes) throws Exception {
		String pid = "";
		String lastPid = "invalid";
		for (int i = 0; i < numberOfCrashes; ++i) {
		// This condition makes sure before we kill the process, the process is running AND
		// the last crash was finished.
		while ("".equals(pid) \|\| lastPid.equals(pid)) {
		pid = getDevice().executeShellCommand("pidof " + processName);
		}
		getDevice().executeShellCommand("kill " + pid);
		lastPid = pid;
		}
		}
		}