Merge "Added high impact rollback in RollbackPackageHealthObserver" into main (e14a3a4b) · Commits · e / os / android_frameworks_base

packages/CrashRecovery/services/java/com/android/server/PackageWatchdog.java

+6 −2

Original line number	Diff line number	Diff line
		@@ -100,13 +100,15 @@ public class PackageWatchdog {
		public static final int FAILURE_REASON_EXPLICIT_HEALTH_CHECK = 2;
		public static final int FAILURE_REASON_APP_CRASH = 3;
		public static final int FAILURE_REASON_APP_NOT_RESPONDING = 4;
		public static final int FAILURE_REASON_BOOT_LOOP = 5;

		@IntDef(prefix = { "FAILURE_REASON_" }, value = {
		FAILURE_REASON_UNKNOWN,
		FAILURE_REASON_NATIVE_CRASH,
		FAILURE_REASON_EXPLICIT_HEALTH_CHECK,
		FAILURE_REASON_APP_CRASH,
		FAILURE_REASON_APP_NOT_RESPONDING
		FAILURE_REASON_APP_NOT_RESPONDING,
		FAILURE_REASON_BOOT_LOOP
		})
		@Retention(RetentionPolicy.SOURCE)
		public @interface FailureReasons {}
		@@ -542,7 +544,7 @@ public class PackageWatchdog {
		mNumberOfNativeCrashPollsRemaining--;
		// Check if native watchdog reported a crash
		if ("1".equals(SystemProperties.get("sys.init.updatable_crashing"))) {
		// We rollback everything available when crash is unattributable
		// We rollback all available low impact rollbacks when crash is unattributable
		onPackageFailure(Collections.EMPTY_LIST, FAILURE_REASON_NATIVE_CRASH);
		// we stop polling after an attempt to execute rollback, regardless of whether the
		// attempt succeeds or not
		@@ -572,6 +574,7 @@ public class PackageWatchdog {
		PackageHealthObserverImpact.USER_IMPACT_LEVEL_30,
		PackageHealthObserverImpact.USER_IMPACT_LEVEL_50,
		PackageHealthObserverImpact.USER_IMPACT_LEVEL_70,
		PackageHealthObserverImpact.USER_IMPACT_LEVEL_90,
		PackageHealthObserverImpact.USER_IMPACT_LEVEL_100})
		public @interface PackageHealthObserverImpact {
		/** No action to take. */
		@@ -582,6 +585,7 @@ public class PackageWatchdog {
		int USER_IMPACT_LEVEL_30 = 30;
		int USER_IMPACT_LEVEL_50 = 50;
		int USER_IMPACT_LEVEL_70 = 70;
		int USER_IMPACT_LEVEL_90 = 90;
		/* Action has high user impact, a last resort, user of a device will be very frustrated. */
		int USER_IMPACT_LEVEL_100 = 100;
		}

packages/CrashRecovery/services/java/com/android/server/rollback/RollbackPackageHealthObserver.java

+252 −48

Original line number	Diff line number	Diff line
		@@ -28,6 +28,7 @@ import android.content.pm.VersionedPackage;
		import android.content.rollback.PackageRollbackInfo;
		import android.content.rollback.RollbackInfo;
		import android.content.rollback.RollbackManager;
		import android.crashrecovery.flags.Flags;
		import android.os.Environment;
		import android.os.FileUtils;
		import android.os.Handler;
		@@ -45,7 +46,6 @@ import com.android.server.PackageWatchdog;
		import com.android.server.PackageWatchdog.FailureReasons;
		import com.android.server.PackageWatchdog.PackageHealthObserver;
		import com.android.server.PackageWatchdog.PackageHealthObserverImpact;
		import com.android.server.SystemConfig;
		import com.android.server.crashrecovery.proto.CrashRecoveryStatsLog;
		import com.android.server.pm.ApexManager;

		@@ -57,6 +57,7 @@ import java.io.FileReader;
		import java.io.IOException;
		import java.io.PrintWriter;
		import java.util.Collections;
		import java.util.Comparator;
		import java.util.List;
		import java.util.Set;
		import java.util.function.Consumer;
		@@ -84,7 +85,8 @@ final class RollbackPackageHealthObserver implements PackageHealthObserver {
		// True if needing to roll back only rebootless apexes when native crash happens
		private boolean mTwoPhaseRollbackEnabled;

		RollbackPackageHealthObserver(Context context) {
		@VisibleForTesting
		RollbackPackageHealthObserver(Context context, ApexManager apexManager) {
		mContext = context;
		HandlerThread handlerThread = new HandlerThread("RollbackPackageHealthObserver");
		handlerThread.start();
		@@ -94,7 +96,7 @@ final class RollbackPackageHealthObserver implements PackageHealthObserver {
		mLastStagedRollbackIdsFile = new File(dataDir, "last-staged-rollback-ids");
		mTwoPhaseRollbackEnabledFile = new File(dataDir, "two-phase-rollback-enabled");
		PackageWatchdog.getInstance(mContext).registerHealthObserver(this);
		mApexManager = ApexManager.getInstance();
		mApexManager = apexManager;

		if (SystemProperties.getBoolean("sys.boot_completed", false)) {
		// Load the value from the file if system server has crashed and restarted
		@@ -107,12 +109,33 @@ final class RollbackPackageHealthObserver implements PackageHealthObserver {
		}
		}

		RollbackPackageHealthObserver(Context context) {
		this(context, ApexManager.getInstance());
		}

		@Override
		public int onHealthCheckFailed(@Nullable VersionedPackage failedPackage,
		@FailureReasons int failureReason, int mitigationCount) {
		int impact = PackageHealthObserverImpact.USER_IMPACT_LEVEL_0;
		if (Flags.recoverabilityDetection()) {
		List<RollbackInfo> availableRollbacks = getAvailableRollbacks();
		List<RollbackInfo> lowImpactRollbacks = getRollbacksAvailableForImpactLevel(
		availableRollbacks, PackageManager.ROLLBACK_USER_IMPACT_LOW);
		if (!lowImpactRollbacks.isEmpty()) {
		if (failureReason == PackageWatchdog.FAILURE_REASON_NATIVE_CRASH) {
		// For native crashes, we will directly roll back any available rollbacks at low
		// impact level
		impact = PackageHealthObserverImpact.USER_IMPACT_LEVEL_30;
		} else if (getRollbackForPackage(failedPackage, lowImpactRollbacks) != null) {
		// Rollback is available for crashing low impact package
		impact = PackageHealthObserverImpact.USER_IMPACT_LEVEL_30;
		} else {
		impact = PackageHealthObserverImpact.USER_IMPACT_LEVEL_70;
		}
		}
		} else {
		boolean anyRollbackAvailable = !mContext.getSystemService(RollbackManager.class)
		.getAvailableRollbacks().isEmpty();
		int impact = PackageHealthObserverImpact.USER_IMPACT_LEVEL_0;

		if (failureReason == PackageWatchdog.FAILURE_REASON_NATIVE_CRASH
		&& anyRollbackAvailable) {
		@@ -126,6 +149,7 @@ final class RollbackPackageHealthObserver implements PackageHealthObserver {
		// If any rollbacks are available, we will commit them
		impact = PackageHealthObserverImpact.USER_IMPACT_LEVEL_70;
		}
		}

		return impact;
		}
		@@ -133,6 +157,23 @@ final class RollbackPackageHealthObserver implements PackageHealthObserver {
		@Override
		public boolean execute(@Nullable VersionedPackage failedPackage,
		@FailureReasons int rollbackReason, int mitigationCount) {
		if (Flags.recoverabilityDetection()) {
		List<RollbackInfo> availableRollbacks = getAvailableRollbacks();
		if (rollbackReason == PackageWatchdog.FAILURE_REASON_NATIVE_CRASH) {
		mHandler.post(() -> rollbackAllLowImpact(availableRollbacks, rollbackReason));
		return true;
		}

		List<RollbackInfo> lowImpactRollbacks = getRollbacksAvailableForImpactLevel(
		availableRollbacks, PackageManager.ROLLBACK_USER_IMPACT_LOW);
		RollbackInfo rollback = getRollbackForPackage(failedPackage, lowImpactRollbacks);
		if (rollback != null) {
		mHandler.post(() -> rollbackPackage(rollback, failedPackage, rollbackReason));
		} else if (!lowImpactRollbacks.isEmpty()) {
		// Apply all available low impact rollbacks.
		mHandler.post(() -> rollbackAllLowImpact(availableRollbacks, rollbackReason));
		}
		} else {
		if (rollbackReason == PackageWatchdog.FAILURE_REASON_NATIVE_CRASH) {
		mHandler.post(() -> rollbackAll(rollbackReason));
		return true;
		@@ -144,11 +185,37 @@ final class RollbackPackageHealthObserver implements PackageHealthObserver {
		} else {
		mHandler.post(() -> rollbackAll(rollbackReason));
		}
		}

		// Assume rollbacks executed successfully
		return true;
		}

		@Override
		public int onBootLoop(int mitigationCount) {
		int impact = PackageHealthObserverImpact.USER_IMPACT_LEVEL_0;
		if (Flags.recoverabilityDetection()) {
		List<RollbackInfo> availableRollbacks = getAvailableRollbacks();
		if (!availableRollbacks.isEmpty()) {
		impact = getUserImpactBasedOnRollbackImpactLevel(availableRollbacks);
		}
		}
		return impact;
		}

		@Override
		public boolean executeBootLoopMitigation(int mitigationCount) {
		if (Flags.recoverabilityDetection()) {
		List<RollbackInfo> availableRollbacks = getAvailableRollbacks();

		triggerLeastImpactLevelRollback(availableRollbacks,
		PackageWatchdog.FAILURE_REASON_BOOT_LOOP);
		return true;
		}
		return false;
		}


		@Override
		public String getName() {
		return NAME;
		@@ -161,13 +228,16 @@ final class RollbackPackageHealthObserver implements PackageHealthObserver {

		@Override
		public boolean mayObservePackage(String packageName) {
		if (mContext.getSystemService(RollbackManager.class)
		.getAvailableRollbacks().isEmpty()) {
		if (getAvailableRollbacks().isEmpty()) {
		return false;
		}
		return isPersistentSystemApp(packageName);
		}

		private List<RollbackInfo> getAvailableRollbacks() {
		return mContext.getSystemService(RollbackManager.class).getAvailableRollbacks();
		}

		private boolean isPersistentSystemApp(@NonNull String packageName) {
		PackageManager pm = mContext.getPackageManager();
		try {
		@@ -272,6 +342,40 @@ final class RollbackPackageHealthObserver implements PackageHealthObserver {
		return null;
		}

		@AnyThread
		private RollbackInfo getRollbackForPackage(@Nullable VersionedPackage failedPackage,
		List<RollbackInfo> availableRollbacks) {
		if (failedPackage == null) {
		return null;
		}

		for (RollbackInfo rollback : availableRollbacks) {
		for (PackageRollbackInfo packageRollback : rollback.getPackages()) {
		if (packageRollback.getVersionRolledBackFrom().equals(failedPackage)) {
		return rollback;
		}
		// TODO(b/147666157): Extract version number of apk-in-apex so that we don't have
		// to rely on complicated reasoning as below

		// Due to b/147666157, for apk in apex, we do not know the version we are rolling
		// back from. But if a package X is embedded in apex A exclusively (not embedded in
		// any other apex), which is not guaranteed, then it is sufficient to check only
		// package names here, as the version of failedPackage and the PackageRollbackInfo
		// can't be different. If failedPackage has a higher version, then it must have
		// been updated somehow. There are two ways: it was updated by an update of apex A
		// or updated directly as apk. In both cases, this rollback would have gotten
		// expired when onPackageReplaced() was called. Since the rollback exists, it has
		// same version as failedPackage.
		if (packageRollback.isApkInApex()
		&& packageRollback.getVersionRolledBackFrom().getPackageName()
		.equals(failedPackage.getPackageName())) {
		return rollback;
		}
		}
		}
		return null;
		}

		/**
		* Returns {@code true} if staged session associated with {@code rollbackId} was marked
		* as handled, {@code false} if already handled.
		@@ -396,12 +500,6 @@ final class RollbackPackageHealthObserver implements PackageHealthObserver {
		@FailureReasons int rollbackReason) {
		assertInWorkerThread();

		if (isAutomaticRollbackDenied(SystemConfig.getInstance(), failedPackage)) {
		Slog.d(TAG, "Automatic rollback not allowed for package "
		+ failedPackage.getPackageName());
		return;
		}

		final RollbackManager rollbackManager = mContext.getSystemService(RollbackManager.class);
		int reasonToLog = WatchdogRollbackLogger.mapFailureReasonToMetric(rollbackReason);
		final String failedPackageToLog;
		@@ -464,17 +562,6 @@ final class RollbackPackageHealthObserver implements PackageHealthObserver {
		Collections.singletonList(failedPackage), rollbackReceiver.getIntentSender());
		}

		/**
		* Returns true if this package is not eligible for automatic rollback.
		*/
		@VisibleForTesting
		@AnyThread
		public static boolean isAutomaticRollbackDenied(SystemConfig systemConfig,
		VersionedPackage versionedPackage) {
		return systemConfig.getAutomaticRollbackDenylistedPackages()
		.contains(versionedPackage.getPackageName());
		}

		/**
		* Two-phase rollback:
		* 1. roll back rebootless apexes first
		@@ -495,14 +582,62 @@ final class RollbackPackageHealthObserver implements PackageHealthObserver {
		boolean found = false;
		for (RollbackInfo rollback : rollbacks) {
		if (isRebootlessApex(rollback)) {
		VersionedPackage sample = rollback.getPackages().get(0).getVersionRolledBackFrom();
		rollbackPackage(rollback, sample, PackageWatchdog.FAILURE_REASON_NATIVE_CRASH);
		VersionedPackage firstRollback =
		rollback.getPackages().get(0).getVersionRolledBackFrom();
		rollbackPackage(rollback, firstRollback,
		PackageWatchdog.FAILURE_REASON_NATIVE_CRASH);
		found = true;
		}
		}
		return found;
		}

		/**
		* Rollback the package that has minimum rollback impact level.
		* @param availableRollbacks all available rollbacks
		* @param rollbackReason reason to rollback
		*/
		private void triggerLeastImpactLevelRollback(List<RollbackInfo> availableRollbacks,
		@FailureReasons int rollbackReason) {
		int minRollbackImpactLevel = getMinRollbackImpactLevel(availableRollbacks);

		if (minRollbackImpactLevel == PackageManager.ROLLBACK_USER_IMPACT_LOW) {
		// Apply all available low impact rollbacks.
		mHandler.post(() -> rollbackAllLowImpact(availableRollbacks, rollbackReason));
		} else if (minRollbackImpactLevel == PackageManager.ROLLBACK_USER_IMPACT_HIGH) {
		// Rollback one package at a time. If that doesn't resolve the issue, rollback
		// next with same impact level.
		mHandler.post(() -> rollbackHighImpact(availableRollbacks, rollbackReason));
		}
		}

		/**
		* sort the available high impact rollbacks by first package name to have a deterministic order.
		* Apply the first available rollback.
		* @param availableRollbacks all available rollbacks
		* @param rollbackReason reason to rollback
		*/
		@WorkerThread
		private void rollbackHighImpact(List<RollbackInfo> availableRollbacks,
		@FailureReasons int rollbackReason) {
		assertInWorkerThread();
		List<RollbackInfo> highImpactRollbacks =
		getRollbacksAvailableForImpactLevel(
		availableRollbacks, PackageManager.ROLLBACK_USER_IMPACT_HIGH);

		// sort rollbacks based on package name of the first package. This is to have a
		// deterministic order of rollbacks.
		List<RollbackInfo> sortedHighImpactRollbacks = highImpactRollbacks.stream().sorted(
		Comparator.comparing(a -> a.getPackages().get(0).getPackageName())).toList();
		VersionedPackage firstRollback =
		sortedHighImpactRollbacks
		.get(0)
		.getPackages()
		.get(0)
		.getVersionRolledBackFrom();
		rollbackPackage(sortedHighImpactRollbacks.get(0), firstRollback, rollbackReason);
		}

		@WorkerThread
		private void rollbackAll(@FailureReasons int rollbackReason) {
		assertInWorkerThread();
		@@ -522,8 +657,77 @@ final class RollbackPackageHealthObserver implements PackageHealthObserver {
		}

		for (RollbackInfo rollback : rollbacks) {
		VersionedPackage sample = rollback.getPackages().get(0).getVersionRolledBackFrom();
		rollbackPackage(rollback, sample, rollbackReason);
		VersionedPackage firstRollback =
		rollback.getPackages().get(0).getVersionRolledBackFrom();
		rollbackPackage(rollback, firstRollback, rollbackReason);
		}
		}

		/**
		* Rollback all available low impact rollbacks
		* @param availableRollbacks all available rollbacks
		* @param rollbackReason reason to rollbacks
		*/
		@WorkerThread
		private void rollbackAllLowImpact(
		List<RollbackInfo> availableRollbacks, @FailureReasons int rollbackReason) {
		assertInWorkerThread();

		List<RollbackInfo> lowImpactRollbacks = getRollbacksAvailableForImpactLevel(
		availableRollbacks,
		PackageManager.ROLLBACK_USER_IMPACT_LOW);
		if (useTwoPhaseRollback(lowImpactRollbacks)) {
		return;
		}

		Slog.i(TAG, "Rolling back all available low impact rollbacks");
		// Add all rollback ids to mPendingStagedRollbackIds, so that we do not reboot before all
		// pending staged rollbacks are handled.
		for (RollbackInfo rollback : lowImpactRollbacks) {
		if (rollback.isStaged()) {
		mPendingStagedRollbackIds.add(rollback.getRollbackId());
		}
		}

		for (RollbackInfo rollback : lowImpactRollbacks) {
		VersionedPackage firstRollback =
		rollback.getPackages().get(0).getVersionRolledBackFrom();
		rollbackPackage(rollback, firstRollback, rollbackReason);
		}
		}

		private List<RollbackInfo> getRollbacksAvailableForImpactLevel(
		List<RollbackInfo> availableRollbacks, int impactLevel) {
		return availableRollbacks.stream()
		.filter(rollbackInfo -> rollbackInfo.getRollbackImpactLevel() == impactLevel)
		.toList();
		}

		private int getMinRollbackImpactLevel(List<RollbackInfo> availableRollbacks) {
		return availableRollbacks.stream()
		.mapToInt(RollbackInfo::getRollbackImpactLevel)
		.min()
		.orElse(-1);
		}

		private int getUserImpactBasedOnRollbackImpactLevel(List<RollbackInfo> availableRollbacks) {
		int impact = PackageHealthObserverImpact.USER_IMPACT_LEVEL_0;
		int minImpact = getMinRollbackImpactLevel(availableRollbacks);
		switch (minImpact) {
		case PackageManager.ROLLBACK_USER_IMPACT_LOW:
		impact = PackageHealthObserverImpact.USER_IMPACT_LEVEL_70;
		break;
		case PackageManager.ROLLBACK_USER_IMPACT_HIGH:
		impact = PackageHealthObserverImpact.USER_IMPACT_LEVEL_90;
		break;
		default:
		impact = PackageHealthObserverImpact.USER_IMPACT_LEVEL_0;
		}
		return impact;
		}

		@VisibleForTesting
		Handler getHandler() {
		return mHandler;
		}
		}

packages/CrashRecovery/services/java/com/android/server/rollback/WatchdogRollbackLogger.java

+3 −0

Original line number	Diff line number	Diff line
		@@ -18,6 +18,7 @@ package com.android.server.rollback;

		import static com.android.server.crashrecovery.proto.CrashRecoveryStatsLog.WATCHDOG_ROLLBACK_OCCURRED__ROLLBACK_REASON__REASON_APP_CRASH;
		import static com.android.server.crashrecovery.proto.CrashRecoveryStatsLog.WATCHDOG_ROLLBACK_OCCURRED__ROLLBACK_REASON__REASON_APP_NOT_RESPONDING;
		import static com.android.server.crashrecovery.proto.CrashRecoveryStatsLog.WATCHDOG_ROLLBACK_OCCURRED__ROLLBACK_REASON__REASON_BOOT_LOOPING;
		import static com.android.server.crashrecovery.proto.CrashRecoveryStatsLog.WATCHDOG_ROLLBACK_OCCURRED__ROLLBACK_REASON__REASON_EXPLICIT_HEALTH_CHECK;
		import static com.android.server.crashrecovery.proto.CrashRecoveryStatsLog.WATCHDOG_ROLLBACK_OCCURRED__ROLLBACK_REASON__REASON_NATIVE_CRASH;
		import static com.android.server.crashrecovery.proto.CrashRecoveryStatsLog.WATCHDOG_ROLLBACK_OCCURRED__ROLLBACK_REASON__REASON_NATIVE_CRASH_DURING_BOOT;
		@@ -258,6 +259,8 @@ public final class WatchdogRollbackLogger {
		return WATCHDOG_ROLLBACK_OCCURRED__ROLLBACK_REASON__REASON_APP_CRASH;
		case PackageWatchdog.FAILURE_REASON_APP_NOT_RESPONDING:
		return WATCHDOG_ROLLBACK_OCCURRED__ROLLBACK_REASON__REASON_APP_NOT_RESPONDING;
		case PackageWatchdog.FAILURE_REASON_BOOT_LOOP:
		return WATCHDOG_ROLLBACK_OCCURRED__ROLLBACK_REASON__REASON_BOOT_LOOPING;
		default:
		return WATCHDOG_ROLLBACK_OCCURRED__ROLLBACK_REASON__REASON_UNKNOWN;
		}

services/core/java/com/android/server/SystemConfig.java

+0 −15

Original line number	Diff line number	Diff line
		@@ -326,7 +326,6 @@ public class SystemConfig {
		private ArrayMap<String, Set<String>> mPackageToUserTypeBlacklist = new ArrayMap<>();

		private final ArraySet<String> mRollbackWhitelistedPackages = new ArraySet<>();
		private final ArraySet<String> mAutomaticRollbackDenylistedPackages = new ArraySet<>();
		private final ArraySet<String> mWhitelistedStagedInstallers = new ArraySet<>();
		// A map from package name of vendor APEXes that can be updated to an installer package name
		// allowed to install updates for it.
		@@ -475,10 +474,6 @@ public class SystemConfig {
		return mRollbackWhitelistedPackages;
		}

		public Set<String> getAutomaticRollbackDenylistedPackages() {
		return mAutomaticRollbackDenylistedPackages;
		}

		public Set<String> getWhitelistedStagedInstallers() {
		return mWhitelistedStagedInstallers;
		}
		@@ -1396,16 +1391,6 @@ public class SystemConfig {
		}
		XmlUtils.skipCurrentTag(parser);
		} break;
		case "automatic-rollback-denylisted-app": {
		String pkgname = parser.getAttributeValue(null, "package");
		if (pkgname == null) {
		Slog.w(TAG, "<" + name + "> without package in " + permFile
		+ " at " + parser.getPositionDescription());
		} else {
		mAutomaticRollbackDenylistedPackages.add(pkgname);
		}
		XmlUtils.skipCurrentTag(parser);
		} break;
		case "whitelisted-staged-installer": {
		if (allowAppConfigs) {
		String pkgname = parser.getAttributeValue(null, "package");

services/core/java/com/android/server/rollback/RollbackManagerServiceImpl.java

+14 −7

Original line number	Diff line number	Diff line
		@@ -1212,6 +1212,8 @@ class RollbackManagerServiceImpl extends IRollbackManager.Stub implements Rollba
		rollback.makeAvailable();
		mPackageHealthObserver.notifyRollbackAvailable(rollback.info);

		if (Flags.recoverabilityDetection()) {
		if (rollback.info.getRollbackImpactLevel() == PackageManager.ROLLBACK_USER_IMPACT_LOW) {
		// TODO(zezeozue): Provide API to explicitly start observing instead
		// of doing this for all rollbacks. If we do this for all rollbacks,
		// should document in PackageInstaller.SessionParams#setEnableRollback
		@@ -1219,6 +1221,11 @@ class RollbackManagerServiceImpl extends IRollbackManager.Stub implements Rollba
		// prepare to rollback if packages crashes too frequently.
		mPackageHealthObserver.startObservingHealth(rollback.getPackageNames(),
		mRollbackLifetimeDurationInMillis);
		}
		} else {
		mPackageHealthObserver.startObservingHealth(rollback.getPackageNames(),
		mRollbackLifetimeDurationInMillis);
		}
		runExpiration();
		}