Add recoverability detection to RescueParty and PackageWatchdog (d473821a) · Commits · e / os / android_frameworks_base

packages/CrashRecovery/services/java/com/android/server/PackageWatchdog.java

+212 −17

Original line number	Diff line number	Diff line
		@@ -26,6 +26,7 @@ import android.content.Context;
		import android.content.pm.PackageInfo;
		import android.content.pm.PackageManager;
		import android.content.pm.VersionedPackage;
		import android.crashrecovery.flags.Flags;
		import android.net.ConnectivityModuleConnector;
		import android.os.Environment;
		import android.os.Handler;
		@@ -57,16 +58,20 @@ import org.xmlpull.v1.XmlPullParserException;
		import java.io.BufferedReader;
		import java.io.BufferedWriter;
		import java.io.File;
		import java.io.FileInputStream;
		import java.io.FileNotFoundException;
		import java.io.FileOutputStream;
		import java.io.FileReader;
		import java.io.FileWriter;
		import java.io.IOException;
		import java.io.InputStream;
		import java.io.ObjectInputStream;
		import java.io.ObjectOutputStream;
		import java.lang.annotation.Retention;
		import java.lang.annotation.RetentionPolicy;
		import java.util.ArrayList;
		import java.util.Collections;
		import java.util.HashMap;
		import java.util.Iterator;
		import java.util.List;
		import java.util.Map;
		@@ -130,8 +135,25 @@ public class PackageWatchdog {

		@VisibleForTesting
		static final int DEFAULT_BOOT_LOOP_TRIGGER_COUNT = 5;
		@VisibleForTesting

		static final long DEFAULT_BOOT_LOOP_TRIGGER_WINDOW_MS = TimeUnit.MINUTES.toMillis(10);
		// Boot loop at which packageWatchdog starts first mitigation
		private static final String BOOT_LOOP_THRESHOLD =
		"persist.device_config.configuration.boot_loop_threshold";
		@VisibleForTesting
		static final int DEFAULT_BOOT_LOOP_THRESHOLD = 15;
		// Once boot_loop_threshold is surpassed next mitigation would be triggered after
		// specified number of reboots.
		private static final String BOOT_LOOP_MITIGATION_INCREMENT =
		"persist.device_config.configuration..boot_loop_mitigation_increment";
		@VisibleForTesting
		static final int DEFAULT_BOOT_LOOP_MITIGATION_INCREMENT = 2;

		// Threshold level at which or above user might experience significant disruption.
		private static final String MAJOR_USER_IMPACT_LEVEL_THRESHOLD =
		"persist.device_config.configuration.major_user_impact_level_threshold";
		private static final int DEFAULT_MAJOR_USER_IMPACT_LEVEL_THRESHOLD =
		PackageHealthObserverImpact.USER_IMPACT_LEVEL_71;

		private long mNumberOfNativeCrashPollsRemaining;

		@@ -145,6 +167,7 @@ public class PackageWatchdog {
		private static final String ATTR_EXPLICIT_HEALTH_CHECK_DURATION = "health-check-duration";
		private static final String ATTR_PASSED_HEALTH_CHECK = "passed-health-check";
		private static final String ATTR_MITIGATION_CALLS = "mitigation-calls";
		private static final String ATTR_MITIGATION_COUNT = "mitigation-count";

		// A file containing information about the current mitigation count in the case of a boot loop.
		// This allows boot loop information to persist in the case of an fs-checkpoint being
		@@ -230,8 +253,16 @@ public class PackageWatchdog {
		mConnectivityModuleConnector = connectivityModuleConnector;
		mSystemClock = clock;
		mNumberOfNativeCrashPollsRemaining = NUMBER_OF_NATIVE_CRASH_POLLS;
		if (Flags.recoverabilityDetection()) {
		mBootThreshold = new BootThreshold(DEFAULT_BOOT_LOOP_TRIGGER_COUNT,
		DEFAULT_BOOT_LOOP_TRIGGER_WINDOW_MS,
		SystemProperties.getInt(BOOT_LOOP_MITIGATION_INCREMENT,
		DEFAULT_BOOT_LOOP_MITIGATION_INCREMENT));
		} else {
		mBootThreshold = new BootThreshold(DEFAULT_BOOT_LOOP_TRIGGER_COUNT,
		DEFAULT_BOOT_LOOP_TRIGGER_WINDOW_MS);
		}

		loadFromFile();
		sPackageWatchdog = this;
		}
		@@ -436,12 +467,17 @@ public class PackageWatchdog {
		mitigationCount =
		currentMonitoredPackage.getMitigationCountLocked();
		}
		if (Flags.recoverabilityDetection()) {
		maybeExecute(currentObserverToNotify, versionedPackage,
		failureReason, currentObserverImpact, mitigationCount);
		} else {
		currentObserverToNotify.execute(versionedPackage,
		failureReason, mitigationCount);
		}
		}
		}
		}
		}
		});
		}

		@@ -467,34 +503,72 @@ public class PackageWatchdog {
		}
		}
		if (currentObserverToNotify != null) {
		if (Flags.recoverabilityDetection()) {
		maybeExecute(currentObserverToNotify, failingPackage, failureReason,
		currentObserverImpact, /mitigationCount=/ 1);
		} else {
		currentObserverToNotify.execute(failingPackage, failureReason, 1);
		}
		}
		}

		private void maybeExecute(PackageHealthObserver currentObserverToNotify,
		VersionedPackage versionedPackage,
		@FailureReasons int failureReason,
		int currentObserverImpact,
		int mitigationCount) {
		if (currentObserverImpact < getUserImpactLevelLimit()) {
		currentObserverToNotify.execute(versionedPackage, failureReason, mitigationCount);
		}
		}


		/**
		* Called when the system server boots. If the system server is detected to be in a boot loop,
		* query each observer and perform the mitigation action with the lowest user impact.
		*/
		@SuppressWarnings("GuardedBy")
		public void noteBoot() {
		synchronized (mLock) {
		if (mBootThreshold.incrementAndTest()) {
		boolean mitigate = mBootThreshold.incrementAndTest();
		if (mitigate) {
		if (!Flags.recoverabilityDetection()) {
		mBootThreshold.reset();
		}
		int mitigationCount = mBootThreshold.getMitigationCount() + 1;
		PackageHealthObserver currentObserverToNotify = null;
		ObserverInternal currentObserverInternal = null;
		int currentObserverImpact = Integer.MAX_VALUE;
		for (int i = 0; i < mAllObservers.size(); i++) {
		final ObserverInternal observer = mAllObservers.valueAt(i);
		PackageHealthObserver registeredObserver = observer.registeredObserver;
		if (registeredObserver != null) {
		int impact = registeredObserver.onBootLoop(mitigationCount);
		int impact = Flags.recoverabilityDetection()
		? registeredObserver.onBootLoop(
		observer.getBootMitigationCount() + 1)
		: registeredObserver.onBootLoop(mitigationCount);
		if (impact != PackageHealthObserverImpact.USER_IMPACT_LEVEL_0
		&& impact < currentObserverImpact) {
		currentObserverToNotify = registeredObserver;
		currentObserverInternal = observer;
		currentObserverImpact = impact;
		}
		}
		}
		if (currentObserverToNotify != null) {
		if (Flags.recoverabilityDetection()) {
		if (currentObserverImpact < getUserImpactLevelLimit()
		\|\| (currentObserverImpact >= getUserImpactLevelLimit()
		&& mBootThreshold.getCount() >= getBootLoopThreshold())) {
		int currentObserverMitigationCount =
		currentObserverInternal.getBootMitigationCount() + 1;
		currentObserverInternal.setBootMitigationCount(
		currentObserverMitigationCount);
		saveAllObserversBootMitigationCountToMetadata(METADATA_FILE);
		currentObserverToNotify.executeBootLoopMitigation(
		currentObserverMitigationCount);
		}
		} else {
		mBootThreshold.setMitigationCount(mitigationCount);
		mBootThreshold.saveMitigationCountToMetadata();
		currentObserverToNotify.executeBootLoopMitigation(mitigationCount);
		@@ -502,6 +576,7 @@ public class PackageWatchdog {
		}
		}
		}
		}

		// TODO(b/120598832): Optimize write? Maybe only write a separate smaller file? Also
		// avoid holding lock?
		@@ -567,13 +642,27 @@ public class PackageWatchdog {
		mShortTaskHandler.post(()->checkAndMitigateNativeCrashes());
		}

		private int getUserImpactLevelLimit() {
		return SystemProperties.getInt(MAJOR_USER_IMPACT_LEVEL_THRESHOLD,
		DEFAULT_MAJOR_USER_IMPACT_LEVEL_THRESHOLD);
		}

		private int getBootLoopThreshold() {
		return SystemProperties.getInt(BOOT_LOOP_THRESHOLD,
		DEFAULT_BOOT_LOOP_THRESHOLD);
		}

		/** Possible severity values of the user impact of a {@link PackageHealthObserver#execute}. */
		@Retention(SOURCE)
		@IntDef(value = {PackageHealthObserverImpact.USER_IMPACT_LEVEL_0,
		PackageHealthObserverImpact.USER_IMPACT_LEVEL_10,
		PackageHealthObserverImpact.USER_IMPACT_LEVEL_20,
		PackageHealthObserverImpact.USER_IMPACT_LEVEL_30,
		PackageHealthObserverImpact.USER_IMPACT_LEVEL_50,
		PackageHealthObserverImpact.USER_IMPACT_LEVEL_70,
		PackageHealthObserverImpact.USER_IMPACT_LEVEL_71,
		PackageHealthObserverImpact.USER_IMPACT_LEVEL_75,
		PackageHealthObserverImpact.USER_IMPACT_LEVEL_80,
		PackageHealthObserverImpact.USER_IMPACT_LEVEL_90,
		PackageHealthObserverImpact.USER_IMPACT_LEVEL_100})
		public @interface PackageHealthObserverImpact {
		@@ -582,11 +671,15 @@ public class PackageWatchdog {
		/* Action has low user impact, user of a device will barely notice. */
		int USER_IMPACT_LEVEL_10 = 10;
		/* Actions having medium user impact, user of a device will likely notice. */
		int USER_IMPACT_LEVEL_20 = 20;
		int USER_IMPACT_LEVEL_30 = 30;
		int USER_IMPACT_LEVEL_50 = 50;
		int USER_IMPACT_LEVEL_70 = 70;
		int USER_IMPACT_LEVEL_90 = 90;
		/* Action has high user impact, a last resort, user of a device will be very frustrated. */
		int USER_IMPACT_LEVEL_71 = 71;
		int USER_IMPACT_LEVEL_75 = 75;
		int USER_IMPACT_LEVEL_80 = 80;
		int USER_IMPACT_LEVEL_90 = 90;
		int USER_IMPACT_LEVEL_100 = 100;
		}

		@@ -1144,6 +1237,12 @@ public class PackageWatchdog {
		}
		}

		@VisibleForTesting
		@GuardedBy("mLock")
		void registerObserverInternal(ObserverInternal observerInternal) {
		mAllObservers.put(observerInternal.name, observerInternal);
		}

		/**
		* Represents an observer monitoring a set of packages along with the failure thresholds for
		* each package.
		@@ -1151,17 +1250,23 @@ public class PackageWatchdog {
		* <p> Note, the PackageWatchdog#mLock must always be held when reading or writing
		* instances of this class.
		*/
		private static class ObserverInternal {
		static class ObserverInternal {
		public final String name;
		@GuardedBy("mLock")
		private final ArrayMap<String, MonitoredPackage> mPackages = new ArrayMap<>();
		@Nullable
		@GuardedBy("mLock")
		public PackageHealthObserver registeredObserver;
		private int mMitigationCount;

		ObserverInternal(String name, List<MonitoredPackage> packages) {
		this(name, packages, /mitigationCount=/ 0);
		}

		ObserverInternal(String name, List<MonitoredPackage> packages, int mitigationCount) {
		this.name = name;
		updatePackagesLocked(packages);
		this.mMitigationCount = mitigationCount;
		}

		/**
		@@ -1173,6 +1278,9 @@ public class PackageWatchdog {
		try {
		out.startTag(null, TAG_OBSERVER);
		out.attribute(null, ATTR_NAME, name);
		if (Flags.recoverabilityDetection()) {
		out.attributeInt(null, ATTR_MITIGATION_COUNT, mMitigationCount);
		}
		for (int i = 0; i < mPackages.size(); i++) {
		MonitoredPackage p = mPackages.valueAt(i);
		p.writeLocked(out);
		@@ -1185,6 +1293,14 @@ public class PackageWatchdog {
		}
		}

		public int getBootMitigationCount() {
		return mMitigationCount;
		}

		public void setBootMitigationCount(int mitigationCount) {
		mMitigationCount = mitigationCount;
		}

		@GuardedBy("mLock")
		public void updatePackagesLocked(List<MonitoredPackage> packages) {
		for (int pIndex = 0; pIndex < packages.size(); pIndex++) {
		@@ -1289,6 +1405,7 @@ public class PackageWatchdog {
		**/
		public static ObserverInternal read(TypedXmlPullParser parser, PackageWatchdog watchdog) {
		String observerName = null;
		int observerMitigationCount = 0;
		if (TAG_OBSERVER.equals(parser.getName())) {
		observerName = parser.getAttributeValue(null, ATTR_NAME);
		if (TextUtils.isEmpty(observerName)) {
		@@ -1299,6 +1416,9 @@ public class PackageWatchdog {
		List<MonitoredPackage> packages = new ArrayList<>();
		int innerDepth = parser.getDepth();
		try {
		if (Flags.recoverabilityDetection()) {
		observerMitigationCount = parser.getAttributeInt(null, ATTR_MITIGATION_COUNT);
		}
		while (XmlUtils.nextElementWithin(parser, innerDepth)) {
		if (TAG_PACKAGE.equals(parser.getName())) {
		try {
		@@ -1319,7 +1439,7 @@ public class PackageWatchdog {
		if (packages.isEmpty()) {
		return null;
		}
		return new ObserverInternal(observerName, packages);
		return new ObserverInternal(observerName, packages, observerMitigationCount);
		}

		/** Dumps information about this observer and the packages it watches. */
		@@ -1679,6 +1799,27 @@ public class PackageWatchdog {
		}
		}

		@GuardedBy("mLock")
		@SuppressWarnings("GuardedBy")
		void saveAllObserversBootMitigationCountToMetadata(String filePath) {
		HashMap<String, Integer> bootMitigationCounts = new HashMap<>();
		for (int i = 0; i < mAllObservers.size(); i++) {
		final ObserverInternal observer = mAllObservers.valueAt(i);
		bootMitigationCounts.put(observer.name, observer.getBootMitigationCount());
		}

		try {
		FileOutputStream fileStream = new FileOutputStream(new File(filePath));
		ObjectOutputStream objectStream = new ObjectOutputStream(fileStream);
		objectStream.writeObject(bootMitigationCounts);
		objectStream.flush();
		objectStream.close();
		fileStream.close();
		} catch (Exception e) {
		Slog.i(TAG, "Could not save observers metadata to file: " + e);
		}
		}

		/**
		* Handles the thresholding logic for system server boots.
		*/
		@@ -1686,10 +1827,16 @@ public class PackageWatchdog {

		private final int mBootTriggerCount;
		private final long mTriggerWindow;
		private final int mBootMitigationIncrement;

		BootThreshold(int bootTriggerCount, long triggerWindow) {
		this(bootTriggerCount, triggerWindow, /bootMitigationIncrement=/ 1);
		}

		BootThreshold(int bootTriggerCount, long triggerWindow, int bootMitigationIncrement) {
		this.mBootTriggerCount = bootTriggerCount;
		this.mTriggerWindow = triggerWindow;
		this.mBootMitigationIncrement = bootMitigationIncrement;
		}

		public void reset() {
		@@ -1761,8 +1908,13 @@ public class PackageWatchdog {


		/** Increments the boot counter, and returns whether the device is bootlooping. */
		@GuardedBy("mLock")
		public boolean incrementAndTest() {
		if (Flags.recoverabilityDetection()) {
		readAllObserversBootMitigationCountIfNecessary(METADATA_FILE);
		} else {
		readMitigationCountFromMetadataIfNecessary();
		}
		final long now = mSystemClock.uptimeMillis();
		if (now - getStart() < 0) {
		Slog.e(TAG, "Window was less than zero. Resetting start to current time.");
		@@ -1770,8 +1922,12 @@ public class PackageWatchdog {
		setMitigationStart(now);
		}
		if (now - getMitigationStart() > DEFAULT_DEESCALATION_WINDOW_MS) {
		setMitigationCount(0);
		setMitigationStart(now);
		if (Flags.recoverabilityDetection()) {
		resetAllObserversBootMitigationCount();
		} else {
		setMitigationCount(0);
		}
		}
		final long window = now - getStart();
		if (window >= mTriggerWindow) {
		@@ -1782,9 +1938,48 @@ public class PackageWatchdog {
		int count = getCount() + 1;
		setCount(count);
		EventLogTags.writeRescueNote(Process.ROOT_UID, count, window);
		if (Flags.recoverabilityDetection()) {
		boolean mitigate = (count >= mBootTriggerCount)
		&& (count - mBootTriggerCount) % mBootMitigationIncrement == 0;
		return mitigate;
		}
		return count >= mBootTriggerCount;
		}
		}

		@GuardedBy("mLock")
		private void resetAllObserversBootMitigationCount() {
		for (int i = 0; i < mAllObservers.size(); i++) {
		final ObserverInternal observer = mAllObservers.valueAt(i);
		observer.setBootMitigationCount(0);
		}
		}

		@GuardedBy("mLock")
		@SuppressWarnings("GuardedBy")
		void readAllObserversBootMitigationCountIfNecessary(String filePath) {
		File metadataFile = new File(filePath);
		if (metadataFile.exists()) {
		try {
		FileInputStream fileStream = new FileInputStream(metadataFile);
		ObjectInputStream objectStream = new ObjectInputStream(fileStream);
		HashMap<String, Integer> bootMitigationCounts =
		(HashMap<String, Integer>) objectStream.readObject();
		objectStream.close();
		fileStream.close();

		for (int i = 0; i < mAllObservers.size(); i++) {
		final ObserverInternal observer = mAllObservers.valueAt(i);
		if (bootMitigationCounts.containsKey(observer.name)) {
		observer.setBootMitigationCount(
		bootMitigationCounts.get(observer.name));
		}
		}
		} catch (Exception e) {
		Slog.i(TAG, "Could not read observer metadata file: " + e);
		}
		}
		}

		}
		}

packages/CrashRecovery/services/java/com/android/server/RescueParty.java

+271 −63

File changed.

Preview size limit exceeded, changes collapsed.

packages/CrashRecovery/services/java/com/android/server/rollback/RollbackPackageHealthObserver.java

+2 −2

Original line number	Diff line number	Diff line
		@@ -69,7 +69,7 @@ import java.util.function.Consumer;
		*
		* @hide
		*/
		final class RollbackPackageHealthObserver implements PackageHealthObserver {
		public final class RollbackPackageHealthObserver implements PackageHealthObserver {
		private static final String TAG = "RollbackPackageHealthObserver";
		private static final String NAME = "rollback-observer";
		private static final int PERSISTENT_MASK = ApplicationInfo.FLAG_PERSISTENT
		@@ -89,7 +89,7 @@ final class RollbackPackageHealthObserver implements PackageHealthObserver {
		private boolean mTwoPhaseRollbackEnabled;

		@VisibleForTesting
		RollbackPackageHealthObserver(Context context, ApexManager apexManager) {
		public RollbackPackageHealthObserver(Context context, ApexManager apexManager) {
		mContext = context;
		HandlerThread handlerThread = new HandlerThread("RollbackPackageHealthObserver");
		handlerThread.start();

services/tests/mockingservicestests/src/com/android/server/RescuePartyTest.java

+304 −4

File changed.

Preview size limit exceeded, changes collapsed.

tests/PackageWatchdog/Android.bp

+2 −0

Original line number	Diff line number	Diff line
		@@ -28,8 +28,10 @@ android_test {
		static_libs: [
		"junit",
		"mockito-target-extended-minus-junit4",
		"flag-junit",
		"frameworks-base-testutils",
		"androidx.test.rules",
		"PlatformProperties",
		"services.core",
		"services.net",
		"truth",