Merge changes from topic "pw-escalation" (9f41f464) · Commits · e / os / android_frameworks_base

services/core/java/com/android/server/PackageWatchdog.java

+114 −23

Original line number	Diff line number	Diff line
		@@ -68,6 +68,7 @@ import java.util.Collections;
		import java.util.Iterator;
		import java.util.List;
		import java.util.Map;
		import java.util.NoSuchElementException;
		import java.util.Set;
		import java.util.concurrent.TimeUnit;

		@@ -117,6 +118,9 @@ public class PackageWatchdog {
		static final int DEFAULT_TRIGGER_FAILURE_COUNT = 5;
		@VisibleForTesting
		static final long DEFAULT_OBSERVING_DURATION_MS = TimeUnit.DAYS.toMillis(2);
		// Sliding window for tracking how many mitigation calls were made for a package.
		@VisibleForTesting
		static final long DEFAULT_DEESCALATION_WINDOW_MS = TimeUnit.HOURS.toMillis(1);
		// Whether explicit health checks are enabled or not
		private static final boolean DEFAULT_EXPLICIT_HEALTH_CHECK_ENABLED = true;

		@@ -388,6 +392,7 @@ public class PackageWatchdog {
		// Observer that will receive failure for versionedPackage
		PackageHealthObserver currentObserverToNotify = null;
		int currentObserverImpact = Integer.MAX_VALUE;
		MonitoredPackage currentMonitoredPackage = null;

		// Find observer with least user impact
		for (int oIndex = 0; oIndex < mAllObservers.size(); oIndex++) {
		@@ -396,19 +401,33 @@ public class PackageWatchdog {
		if (registeredObserver != null
		&& observer.onPackageFailureLocked(
		versionedPackage.getPackageName())) {
		MonitoredPackage p = observer.getMonitoredPackage(
		versionedPackage.getPackageName());
		int mitigationCount = 1;
		if (p != null) {
		mitigationCount = p.getMitigationCountLocked() + 1;
		}
		int impact = registeredObserver.onHealthCheckFailed(
		versionedPackage, failureReason);
		versionedPackage, failureReason, mitigationCount);
		if (impact != PackageHealthObserverImpact.USER_IMPACT_NONE
		&& impact < currentObserverImpact) {
		currentObserverToNotify = registeredObserver;
		currentObserverImpact = impact;
		currentMonitoredPackage = p;
		}
		}
		}

		// Execute action with least user impact
		if (currentObserverToNotify != null) {
		currentObserverToNotify.execute(versionedPackage, failureReason);
		int mitigationCount = 1;
		if (currentMonitoredPackage != null) {
		currentMonitoredPackage.noteMitigationCallLocked();
		mitigationCount =
		currentMonitoredPackage.getMitigationCountLocked();
		}
		currentObserverToNotify.execute(versionedPackage,
		failureReason, mitigationCount);
		}
		}
		}
		@@ -429,7 +448,7 @@ public class PackageWatchdog {
		PackageHealthObserver registeredObserver = observer.registeredObserver;
		if (registeredObserver != null) {
		int impact = registeredObserver.onHealthCheckFailed(
		failingPackage, failureReason);
		failingPackage, failureReason, 1);
		if (impact != PackageHealthObserverImpact.USER_IMPACT_NONE
		&& impact < currentObserverImpact) {
		currentObserverToNotify = registeredObserver;
		@@ -438,7 +457,7 @@ public class PackageWatchdog {
		}
		}
		if (currentObserverToNotify != null) {
		currentObserverToNotify.execute(failingPackage, failureReason);
		currentObserverToNotify.execute(failingPackage, failureReason, 1);
		}
		}

		@@ -559,6 +578,8 @@ public class PackageWatchdog {
		* @param versionedPackage the package that is failing. This may be null if a native
		* service is crashing.
		* @param failureReason the type of failure that is occurring.
		* @param mitigationCount the number of times mitigation has been called for this package
		* (including this time).
		*
		*
		* @return any one of {@link PackageHealthObserverImpact} to express the impact
		@@ -566,7 +587,8 @@ public class PackageWatchdog {
		*/
		@PackageHealthObserverImpact int onHealthCheckFailed(
		@Nullable VersionedPackage versionedPackage,
		@FailureReasons int failureReason);
		@FailureReasons int failureReason,
		int mitigationCount);

		/**
		* Executes mitigation for {@link #onHealthCheckFailed}.
		@@ -574,10 +596,12 @@ public class PackageWatchdog {
		* @param versionedPackage the package that is failing. This may be null if a native
		* service is crashing.
		* @param failureReason the type of failure that is occurring.
		* @param mitigationCount the number of times mitigation has been called for this package
		* (including this time).
		* @return {@code true} if action was executed successfully, {@code false} otherwise
		*/
		boolean execute(@Nullable VersionedPackage versionedPackage,
		@FailureReasons int failureReason);
		@FailureReasons int failureReason, int mitigationCount);


		/**
		@@ -684,7 +708,7 @@ public class PackageWatchdog {
		synchronized (mLock) {
		for (int observerIdx = 0; observerIdx < mAllObservers.size(); observerIdx++) {
		ObserverInternal observer = mAllObservers.valueAt(observerIdx);
		MonitoredPackage monitoredPackage = observer.packages.get(packageName);
		MonitoredPackage monitoredPackage = observer.getMonitoredPackage(packageName);

		if (monitoredPackage != null) {
		int oldState = monitoredPackage.getHealthCheckStateLocked();
		@@ -713,7 +737,8 @@ public class PackageWatchdog {
		Slog.d(TAG, "Received supported packages " + supportedPackages);
		Iterator<ObserverInternal> oit = mAllObservers.values().iterator();
		while (oit.hasNext()) {
		Iterator<MonitoredPackage> pit = oit.next().packages.values().iterator();
		Iterator<MonitoredPackage> pit = oit.next().getMonitoredPackages()
		.values().iterator();
		while (pit.hasNext()) {
		MonitoredPackage monitoredPackage = pit.next();
		String packageName = monitoredPackage.getName();
		@@ -746,7 +771,7 @@ public class PackageWatchdog {
		while (oit.hasNext()) {
		ObserverInternal observer = oit.next();
		Iterator<MonitoredPackage> pit =
		observer.packages.values().iterator();
		observer.getMonitoredPackages().values().iterator();
		while (pit.hasNext()) {
		MonitoredPackage monitoredPackage = pit.next();
		String packageName = monitoredPackage.getName();
		@@ -804,7 +829,8 @@ public class PackageWatchdog {
		private long getNextStateSyncMillisLocked() {
		long shortestDurationMs = Long.MAX_VALUE;
		for (int oIndex = 0; oIndex < mAllObservers.size(); oIndex++) {
		ArrayMap<String, MonitoredPackage> packages = mAllObservers.valueAt(oIndex).packages;
		ArrayMap<String, MonitoredPackage> packages = mAllObservers.valueAt(oIndex)
		.getMonitoredPackages();
		for (int pIndex = 0; pIndex < packages.size(); pIndex++) {
		MonitoredPackage mp = packages.valueAt(pIndex);
		long duration = mp.getShortestScheduleDurationMsLocked();
		@@ -838,7 +864,7 @@ public class PackageWatchdog {
		if (!failedPackages.isEmpty()) {
		onHealthCheckFailed(observer, failedPackages);
		}
		if (observer.packages.isEmpty() && (observer.registeredObserver == null
		if (observer.getMonitoredPackages().isEmpty() && (observer.registeredObserver == null
		\|\| !observer.registeredObserver.isPersistent())) {
		Slog.i(TAG, "Discarding observer " + observer.name + ". All packages expired");
		it.remove();
		@@ -857,7 +883,7 @@ public class PackageWatchdog {
		VersionedPackage versionedPkg = it.next().mPackage;
		Slog.i(TAG, "Explicit health check failed for package " + versionedPkg);
		registeredObserver.execute(versionedPkg,
		PackageWatchdog.FAILURE_REASON_EXPLICIT_HEALTH_CHECK);
		PackageWatchdog.FAILURE_REASON_EXPLICIT_HEALTH_CHECK, 1);
		}
		}
		}
		@@ -1054,7 +1080,7 @@ public class PackageWatchdog {
		private static class ObserverInternal {
		public final String name;
		@GuardedBy("mLock")
		public final ArrayMap<String, MonitoredPackage> packages = new ArrayMap<>();
		private final ArrayMap<String, MonitoredPackage> mPackages = new ArrayMap<>();
		@Nullable
		@GuardedBy("mLock")
		public PackageHealthObserver registeredObserver;
		@@ -1073,8 +1099,8 @@ public class PackageWatchdog {
		try {
		out.startTag(null, TAG_OBSERVER);
		out.attribute(null, ATTR_NAME, name);
		for (int i = 0; i < packages.size(); i++) {
		MonitoredPackage p = packages.valueAt(i);
		for (int i = 0; i < mPackages.size(); i++) {
		MonitoredPackage p = mPackages.valueAt(i);
		p.writeLocked(out);
		}
		out.endTag(null, TAG_OBSERVER);
		@@ -1089,11 +1115,11 @@ public class PackageWatchdog {
		public void updatePackagesLocked(List<MonitoredPackage> packages) {
		for (int pIndex = 0; pIndex < packages.size(); pIndex++) {
		MonitoredPackage p = packages.get(pIndex);
		MonitoredPackage existingPackage = this.packages.get(p.getName());
		MonitoredPackage existingPackage = getMonitoredPackage(p.getName());
		if (existingPackage != null) {
		existingPackage.updateHealthCheckDuration(p.mDurationMs);
		} else {
		this.packages.put(p.getName(), p);
		putMonitoredPackage(p);
		}
		}
		}
		@@ -1111,7 +1137,7 @@ public class PackageWatchdog {
		@GuardedBy("mLock")
		private Set<MonitoredPackage> prunePackagesLocked(long elapsedMs) {
		Set<MonitoredPackage> failedPackages = new ArraySet<>();
		Iterator<MonitoredPackage> it = packages.values().iterator();
		Iterator<MonitoredPackage> it = mPackages.values().iterator();
		while (it.hasNext()) {
		MonitoredPackage p = it.next();
		int oldState = p.getHealthCheckStateLocked();
		@@ -1134,18 +1160,52 @@ public class PackageWatchdog {
		*/
		@GuardedBy("mLock")
		public boolean onPackageFailureLocked(String packageName) {
		if (packages.get(packageName) == null && registeredObserver.isPersistent()
		if (getMonitoredPackage(packageName) == null && registeredObserver.isPersistent()
		&& registeredObserver.mayObservePackage(packageName)) {
		packages.put(packageName, sPackageWatchdog.newMonitoredPackage(
		putMonitoredPackage(sPackageWatchdog.newMonitoredPackage(
		packageName, DEFAULT_OBSERVING_DURATION_MS, false));
		}
		MonitoredPackage p = packages.get(packageName);
		MonitoredPackage p = getMonitoredPackage(packageName);
		if (p != null) {
		return p.onFailureLocked();
		}
		return false;
		}

		/**
		* Returns the map of packages monitored by this observer.
		*
		* @return a mapping of package names to {@link MonitoredPackage} objects.
		*/
		@GuardedBy("mLock")
		public ArrayMap<String, MonitoredPackage> getMonitoredPackages() {
		return mPackages;
		}

		/**
		* Returns the {@link MonitoredPackage} associated with a given package name if the
		* package is being monitored by this observer.
		*
		* @param packageName: the name of the package.
		* @return the {@link MonitoredPackage} object associated with the package name if one
		* exists, {@code null} otherwise.
		*/
		@GuardedBy("mLock")
		@Nullable
		public MonitoredPackage getMonitoredPackage(String packageName) {
		return mPackages.get(packageName);
		}

		/**
		* Associates a {@link MonitoredPackage} with the observer.
		*
		* @param p: the {@link MonitoredPackage} to store.
		*/
		@GuardedBy("mLock")
		public void putMonitoredPackage(MonitoredPackage p) {
		mPackages.put(p.getName(), p);
		}

		/**
		* Returns one ObserverInternal from the {@code parser} and advances its state.
		*
		@@ -1201,8 +1261,8 @@ public class PackageWatchdog {
		public void dump(IndentingPrintWriter pw) {
		boolean isPersistent = registeredObserver != null && registeredObserver.isPersistent();
		pw.println("Persistent: " + isPersistent);
		for (String packageName : packages.keySet()) {
		MonitoredPackage p = packages.get(packageName);
		for (String packageName : mPackages.keySet()) {
		MonitoredPackage p = getMonitoredPackage(packageName);
		pw.println(packageName + ": ");
		pw.increaseIndent();
		pw.println("# Failures: " + p.mFailureHistory.size());
		@@ -1257,6 +1317,10 @@ public class PackageWatchdog {
		// Times when package failures happen sorted in ascending order
		@GuardedBy("mLock")
		private final LongArrayQueue mFailureHistory = new LongArrayQueue();
		// Times when an observer was called to mitigate this package's failure. Sorted in
		// ascending order.
		@GuardedBy("mLock")
		private final LongArrayQueue mMitigationCalls = new LongArrayQueue();
		// One of STATE_[ACTIVE\|INACTIVE\|PASSED\|FAILED]. Updated on construction and after
		// methods that could change the health check state: handleElapsedTimeLocked and
		// tryPassHealthCheckLocked
		@@ -1321,6 +1385,33 @@ public class PackageWatchdog {
		return failed;
		}

		/**
		* Notes the timestamp of a mitigation call into the observer.
		*/
		@GuardedBy("mLock")
		public void noteMitigationCallLocked() {
		mMitigationCalls.addLast(mSystemClock.uptimeMillis());
		}

		/**
		* Prunes any mitigation calls outside of the de-escalation window, and returns the
		* number of calls that are in the window afterwards.
		*
		* @return the number of mitigation calls made in the de-escalation window.
		*/
		@GuardedBy("mLock")
		public int getMitigationCountLocked() {
		try {
		final long now = mSystemClock.uptimeMillis();
		while (now - mMitigationCalls.peekFirst() > DEFAULT_DEESCALATION_WINDOW_MS) {
		mMitigationCalls.removeFirst();
		}
		} catch (NoSuchElementException ignore) {
		}

		return mMitigationCalls.size();
		}

		/**
		* Sets the initial health check duration.
		*

services/core/java/com/android/server/RescueParty.java

+42 −25

Original line number	Diff line number	Diff line
		@@ -77,6 +77,7 @@ public class RescueParty {
		static final String PROP_ENABLE_RESCUE = "persist.sys.enable_rescue";
		@VisibleForTesting
		static final String PROP_RESCUE_LEVEL = "sys.rescue_level";
		static final String PROP_ATTEMPTING_FACTORY_RESET = "sys.attempting_factory_reset";
		@VisibleForTesting
		static final int LEVEL_NONE = 0;
		@VisibleForTesting
		@@ -155,7 +156,7 @@ public class RescueParty {
		* Check if we're currently attempting to reboot for a factory reset.
		*/
		public static boolean isAttemptingFactoryReset() {
		return SystemProperties.getInt(PROP_RESCUE_LEVEL, LEVEL_NONE) == LEVEL_FACTORY_RESET;
		return SystemProperties.getBoolean(PROP_ATTEMPTING_FACTORY_RESET, false);
		}

		/**
		@@ -230,14 +231,38 @@ public class RescueParty {
		}
		}

		private static int getMaxRescueLevel() {
		return SystemProperties.getBoolean(PROP_DISABLE_FACTORY_RESET_FLAG, false)
		? LEVEL_RESET_SETTINGS_TRUSTED_DEFAULTS : LEVEL_FACTORY_RESET;
		}

		/**
		* Get the rescue level to perform if this is the n-th attempt at mitigating failure.
		*
		* @param mitigationCount: the mitigation attempt number (1 = first attempt etc.)
		* @return the rescue level for the n-th mitigation attempt.
		*/
		private static int getRescueLevel(int mitigationCount) {
		if (mitigationCount == 1) {
		return LEVEL_RESET_SETTINGS_UNTRUSTED_DEFAULTS;
		} else if (mitigationCount == 2) {
		return LEVEL_RESET_SETTINGS_UNTRUSTED_CHANGES;
		} else if (mitigationCount == 3) {
		return LEVEL_RESET_SETTINGS_TRUSTED_DEFAULTS;
		} else if (mitigationCount >= 4) {
		return getMaxRescueLevel();
		} else {
		Slog.w(TAG, "Expected positive mitigation count, was " + mitigationCount);
		return LEVEL_NONE;
		}
		}

		/**
		* Get the next rescue level. This indicates the next level of mitigation that may be taken.
		*/
		private static int getNextRescueLevel() {
		int maxRescueLevel = SystemProperties.getBoolean(PROP_DISABLE_FACTORY_RESET_FLAG, false)
		? LEVEL_RESET_SETTINGS_TRUSTED_DEFAULTS : LEVEL_FACTORY_RESET;
		return MathUtils.constrain(SystemProperties.getInt(PROP_RESCUE_LEVEL, LEVEL_NONE) + 1,
		LEVEL_NONE, maxRescueLevel);
		LEVEL_NONE, getMaxRescueLevel());
		}

		/**
		@@ -256,7 +281,11 @@ public class RescueParty {
		private static void executeRescueLevel(Context context, @Nullable String failedPackage) {
		final int level = SystemProperties.getInt(PROP_RESCUE_LEVEL, LEVEL_NONE);
		if (level == LEVEL_NONE) return;
		executeRescueLevel(context, failedPackage, level);
		}

		private static void executeRescueLevel(Context context, @Nullable String failedPackage,
		int level) {
		Slog.w(TAG, "Attempting rescue level " + levelToString(level));
		try {
		executeRescueLevelInternal(context, level, failedPackage);
		@@ -284,6 +313,7 @@ public class RescueParty {
		case LEVEL_FACTORY_RESET:
		// Request the reboot from a separate thread to avoid deadlock on PackageWatchdog
		// when device shutting down.
		SystemProperties.set(PROP_ATTEMPTING_FACTORY_RESET, "true");
		Runnable runnable = new Runnable() {
		@Override
		public void run() {
		@@ -320,15 +350,6 @@ public class RescueParty {
		}
		}

		private static int getPackageUid(Context context, String packageName) {
		try {
		return context.getPackageManager().getPackageUid(packageName, 0);
		} catch (PackageManager.NameNotFoundException e) {
		// Since UIDs are always >= 0, this value means the UID could not be determined.
		return -1;
		}
		}

		private static void resetAllSettings(Context context, int mode, @Nullable String failedPackage)
		throws Exception {
		// Try our best to reset all settings possible, and once finished
		@@ -359,7 +380,7 @@ public class RescueParty {

		private static void resetDeviceConfig(Context context, int resetMode,
		@Nullable String failedPackage) {
		if (!shouldPerformScopedResets() \|\| failedPackage == null) {
		if (!shouldPerformScopedResets(resetMode) \|\| failedPackage == null) {
		resetAllAffectedNamespaces(context, resetMode);
		} else {
		performScopedReset(context, resetMode, failedPackage);
		@@ -384,11 +405,8 @@ public class RescueParty {
		}
		}

		private static boolean shouldPerformScopedResets() {
		int rescueLevel = MathUtils.constrain(
		SystemProperties.getInt(PROP_RESCUE_LEVEL, LEVEL_NONE),
		LEVEL_NONE, LEVEL_FACTORY_RESET);
		return rescueLevel <= LEVEL_RESET_SETTINGS_UNTRUSTED_CHANGES;
		private static boolean shouldPerformScopedResets(int resetMode) {
		return resetMode <= Settings.RESET_MODE_UNTRUSTED_CHANGES;
		}

		private static void performScopedReset(Context context, int resetMode,
		@@ -452,10 +470,10 @@ public class RescueParty {

		@Override
		public int onHealthCheckFailed(@Nullable VersionedPackage failedPackage,
		@FailureReasons int failureReason) {
		@FailureReasons int failureReason, int mitigationCount) {
		if (!isDisabled() && (failureReason == PackageWatchdog.FAILURE_REASON_APP_CRASH
		\|\| failureReason == PackageWatchdog.FAILURE_REASON_APP_NOT_RESPONDING)) {
		return mapRescueLevelToUserImpact(getNextRescueLevel());
		return mapRescueLevelToUserImpact(getRescueLevel(mitigationCount));
		} else {
		return PackageHealthObserverImpact.USER_IMPACT_NONE;
		}
		@@ -463,16 +481,15 @@ public class RescueParty {

		@Override
		public boolean execute(@Nullable VersionedPackage failedPackage,
		@FailureReasons int failureReason) {
		@FailureReasons int failureReason, int mitigationCount) {
		if (isDisabled()) {
		return false;
		}
		if (failureReason == PackageWatchdog.FAILURE_REASON_APP_CRASH
		\|\| failureReason == PackageWatchdog.FAILURE_REASON_APP_NOT_RESPONDING) {
		int triggerUid = getPackageUid(mContext, failedPackage.getPackageName());
		incrementRescueLevel(triggerUid);
		final int level = getRescueLevel(mitigationCount);
		executeRescueLevel(mContext,
		failedPackage == null ? null : failedPackage.getPackageName());
		failedPackage == null ? null : failedPackage.getPackageName(), level);
		return true;
		} else {
		return false;

services/core/java/com/android/server/rollback/RollbackPackageHealthObserver.java

+2 −2

Original line number	Diff line number	Diff line
		@@ -92,7 +92,7 @@ final class RollbackPackageHealthObserver implements PackageHealthObserver {

		@Override
		public int onHealthCheckFailed(@Nullable VersionedPackage failedPackage,
		@FailureReasons int failureReason) {
		@FailureReasons int failureReason, int mitigationCount) {
		// For native crashes, we will roll back any available rollbacks
		if (failureReason == PackageWatchdog.FAILURE_REASON_NATIVE_CRASH
		&& !mContext.getSystemService(RollbackManager.class)
		@@ -110,7 +110,7 @@ final class RollbackPackageHealthObserver implements PackageHealthObserver {

		@Override
		public boolean execute(@Nullable VersionedPackage failedPackage,
		@FailureReasons int rollbackReason) {
		@FailureReasons int rollbackReason, int mitigationCount) {
		if (rollbackReason == PackageWatchdog.FAILURE_REASON_NATIVE_CRASH) {
		mHandler.post(() -> rollbackAll());
		return true;

services/tests/mockingservicestests/src/com/android/server/RescuePartyTest.java

+20 −67

File changed.

Preview size limit exceeded, changes collapsed.

tests/PackageWatchdog/src/com/android/server/PackageWatchdogTest.java

+46 −3

File changed.

Preview size limit exceeded, changes collapsed.