Merge "Integrate Rescue Party with Package Watchdog" (3df3e3b0) · Commits · e / os / android_frameworks_base

services/core/java/com/android/server/PackageWatchdog.java

+2 −1

Original line number	Original line	Diff line number	Diff line
	@@ -199,13 +199,14 @@ public class PackageWatchdog {
	mSystemClock = clock;		mSystemClock = clock;
	mNumberOfNativeCrashPollsRemaining = NUMBER_OF_NATIVE_CRASH_POLLS;		mNumberOfNativeCrashPollsRemaining = NUMBER_OF_NATIVE_CRASH_POLLS;
	loadFromFile();		loadFromFile();
			sPackageWatchdog = this;
	}		}

	/** Creates or gets singleton instance of PackageWatchdog. */		/** Creates or gets singleton instance of PackageWatchdog. */
	public static PackageWatchdog getInstance(Context context) {		public static PackageWatchdog getInstance(Context context) {
	synchronized (PackageWatchdog.class) {		synchronized (PackageWatchdog.class) {
	if (sPackageWatchdog == null) {		if (sPackageWatchdog == null) {
	sPackageWatchdog = new PackageWatchdog(context);		new PackageWatchdog(context);
	}		}
	return sPackageWatchdog;		return sPackageWatchdog;
	}		}

services/core/java/com/android/server/RescueParty.java

+126 −46

Original line number	Original line	Diff line number	Diff line
	@@ -18,8 +18,12 @@ package com.android.server;

	import static com.android.server.pm.PackageManagerServiceUtils.logCriticalInfo;		import static com.android.server.pm.PackageManagerServiceUtils.logCriticalInfo;

			import android.annotation.Nullable;
	import android.content.ContentResolver;		import android.content.ContentResolver;
	import android.content.Context;		import android.content.Context;
			import android.content.pm.ApplicationInfo;
			import android.content.pm.PackageManager;
			import android.content.pm.VersionedPackage;
	import android.os.Build;		import android.os.Build;
	import android.os.Environment;		import android.os.Environment;
	import android.os.FileUtils;		import android.os.FileUtils;
	@@ -36,8 +40,12 @@ import android.util.Slog;
	import android.util.SparseArray;		import android.util.SparseArray;
	import android.util.StatsLog;		import android.util.StatsLog;

			import com.android.internal.annotations.GuardedBy;
	import com.android.internal.annotations.VisibleForTesting;		import com.android.internal.annotations.VisibleForTesting;
	import com.android.internal.util.ArrayUtils;		import com.android.internal.util.ArrayUtils;
			import com.android.server.PackageWatchdog.FailureReasons;
			import com.android.server.PackageWatchdog.PackageHealthObserver;
			import com.android.server.PackageWatchdog.PackageHealthObserverImpact;
	import com.android.server.am.SettingsToPropertiesMapper;		import com.android.server.am.SettingsToPropertiesMapper;
	import com.android.server.utils.FlagNamespaceUtils;		import com.android.server.utils.FlagNamespaceUtils;

	@@ -79,19 +87,30 @@ public class RescueParty {
	@VisibleForTesting		@VisibleForTesting
	static final long BOOT_TRIGGER_WINDOW_MILLIS = 600 * DateUtils.SECOND_IN_MILLIS;		static final long BOOT_TRIGGER_WINDOW_MILLIS = 600 * DateUtils.SECOND_IN_MILLIS;
	@VisibleForTesting		@VisibleForTesting
	static final long PERSISTENT_APP_CRASH_TRIGGER_WINDOW_MILLIS = 30 * DateUtils.SECOND_IN_MILLIS;
	@VisibleForTesting
	static final String TAG = "RescueParty";		static final String TAG = "RescueParty";

			private static final String NAME = "rescue-party-observer";


	private static final String PROP_DISABLE_RESCUE = "persist.sys.disable_rescue";		private static final String PROP_DISABLE_RESCUE = "persist.sys.disable_rescue";
	private static final String PROP_RESCUE_BOOT_START = "sys.rescue_boot_start";		private static final String PROP_RESCUE_BOOT_START = "sys.rescue_boot_start";
	private static final String PROP_VIRTUAL_DEVICE = "ro.hardware.virtual_device";		private static final String PROP_VIRTUAL_DEVICE = "ro.hardware.virtual_device";

			private static final int PERSISTENT_MASK = ApplicationInfo.FLAG_PERSISTENT
			\| ApplicationInfo.FLAG_SYSTEM;


	/** Threshold for boot loops */		/** Threshold for boot loops */
	private static final Threshold sBoot = new BootThreshold();		private static final Threshold sBoot = new BootThreshold();
	/** Threshold for app crash loops */		/** Threshold for app crash loops */
	private static SparseArray<Threshold> sApps = new SparseArray<>();		private static SparseArray<Threshold> sApps = new SparseArray<>();

			/** Register the Rescue Party observer as a Package Watchdog health observer */
			public static void registerHealthObserver(Context context) {
			PackageWatchdog.getInstance(context).registerHealthObserver(
			RescuePartyObserver.getInstance(context));
			}

	private static boolean isDisabled() {		private static boolean isDisabled() {
	// Check if we're explicitly enabled for testing		// Check if we're explicitly enabled for testing
	if (SystemProperties.getBoolean(PROP_ENABLE_RESCUE, false)) {		if (SystemProperties.getBoolean(PROP_ENABLE_RESCUE, false)) {
	@@ -134,24 +153,6 @@ public class RescueParty {
	}		}
	}		}

	/**
	* Take note of a persistent app or apex module crash. If we notice too many of these
	* events happening in rapid succession, we'll send out a rescue party.
	*/
	public static void noteAppCrash(Context context, int uid) {
	if (isDisabled()) return;
	Threshold t = sApps.get(uid);
	if (t == null) {
	t = new AppThreshold(uid);
	sApps.put(uid, t);
	}
	if (t.incrementAndTest()) {
	t.reset();
	incrementRescueLevel(t.uid);
	executeRescueLevel(context);
	}
	}

	/**		/**
	* Check if we're currently attempting to reboot for a factory reset.		* Check if we're currently attempting to reboot for a factory reset.
	*/		*/
	@@ -171,11 +172,6 @@ public class RescueParty {
	@VisibleForTesting		@VisibleForTesting
	static void resetAllThresholds() {		static void resetAllThresholds() {
	sBoot.reset();		sBoot.reset();

	for (int i = 0; i < sApps.size(); i++) {
	Threshold appThreshold = sApps.get(sApps.keyAt(i));
	appThreshold.reset();
	}
	}		}

	@VisibleForTesting		@VisibleForTesting
	@@ -243,6 +239,28 @@ public class RescueParty {
	FlagNamespaceUtils.NAMESPACE_NO_PACKAGE);		FlagNamespaceUtils.NAMESPACE_NO_PACKAGE);
	}		}

			private static int mapRescueLevelToUserImpact(int rescueLevel) {
			switch(rescueLevel) {
			case LEVEL_RESET_SETTINGS_UNTRUSTED_DEFAULTS:
			case LEVEL_RESET_SETTINGS_UNTRUSTED_CHANGES:
			return PackageHealthObserverImpact.USER_IMPACT_LOW;
			case LEVEL_RESET_SETTINGS_TRUSTED_DEFAULTS:
			case LEVEL_FACTORY_RESET:
			return PackageHealthObserverImpact.USER_IMPACT_HIGH;
			default:
			return PackageHealthObserverImpact.USER_IMPACT_NONE;
			}
			}

			private static int getPackageUid(Context context, String packageName) {
			try {
			return context.getPackageManager().getPackageUid(packageName, 0);
			} catch (PackageManager.NameNotFoundException e) {
			// Since UIDs are always >= 0, this value means the UID could not be determined.
			return -1;
			}
			}

	private static void resetAllSettings(Context context, int mode) throws Exception {		private static void resetAllSettings(Context context, int mode) throws Exception {
	// Try our best to reset all settings possible, and once finished		// Try our best to reset all settings possible, and once finished
	// rethrow any exception that we encountered		// rethrow any exception that we encountered
	@@ -270,6 +288,89 @@ public class RescueParty {
	}		}
	}		}

			/**
			* Handle mitigation action for package failures. This observer will be register to Package
			* Watchdog and will receive calls about package failures. This observer is persistent so it
			* may choose to mitigate failures for packages it has not explicitly asked to observe.
			*/
			public static class RescuePartyObserver implements PackageHealthObserver {

			private Context mContext;

			@GuardedBy("RescuePartyObserver.class")
			static RescuePartyObserver sRescuePartyObserver;

			private RescuePartyObserver(Context context) {
			mContext = context;
			}

			/** Creates or gets singleton instance of RescueParty. */
			public static RescuePartyObserver getInstance(Context context) {
			synchronized (RescuePartyObserver.class) {
			if (sRescuePartyObserver == null) {
			sRescuePartyObserver = new RescuePartyObserver(context);
			}
			return sRescuePartyObserver;
			}
			}

			@Override
			public int onHealthCheckFailed(@Nullable VersionedPackage failedPackage,
			@FailureReasons int failureReason) {
			if (failureReason == PackageWatchdog.FAILURE_REASON_APP_CRASH
			\|\| failureReason == PackageWatchdog.FAILURE_REASON_APP_NOT_RESPONDING) {
			int rescueLevel = MathUtils.constrain(
			SystemProperties.getInt(PROP_RESCUE_LEVEL, LEVEL_NONE) + 1,
			LEVEL_NONE, LEVEL_FACTORY_RESET);
			return mapRescueLevelToUserImpact(rescueLevel);
			} else {
			return PackageHealthObserverImpact.USER_IMPACT_NONE;
			}
			}

			@Override
			public boolean execute(@Nullable VersionedPackage failedPackage,
			@FailureReasons int failureReason) {
			if (isDisabled()) {
			return false;
			}
			if (failureReason == PackageWatchdog.FAILURE_REASON_APP_CRASH
			\|\| failureReason == PackageWatchdog.FAILURE_REASON_APP_NOT_RESPONDING) {
			int triggerUid = getPackageUid(mContext, failedPackage.getPackageName());
			incrementRescueLevel(triggerUid);
			executeRescueLevel(mContext);
			return true;
			} else {
			return false;
			}
			}

			@Override
			public boolean isPersistent() {
			return true;
			}

			@Override
			public boolean mayObservePackage(String packageName) {
			PackageManager pm = mContext.getPackageManager();
			try {
			// A package is a Mainline module if this is non-null
			if (pm.getModuleInfo(packageName, 0) != null) {
			return true;
			}
			ApplicationInfo info = pm.getApplicationInfo(packageName, 0);
			return (info.flags & PERSISTENT_MASK) == PERSISTENT_MASK;
			} catch (PackageManager.NameNotFoundException e) {
			return false;
			}
			}

			@Override
			public String getName() {
			return NAME;
			}
			}

	/**		/**
	* Threshold that can be triggered if a number of events occur within a		* Threshold that can be triggered if a number of events occur within a
	* window of time.		* window of time.
	@@ -349,27 +450,6 @@ public class RescueParty {
	}		}
	}		}

	/**
	* Specialization of {@link Threshold} for monitoring app crashes. It stores
	* counters in memory.
	*/
	private static class AppThreshold extends Threshold {
	private int count;
	private long start;

	public AppThreshold(int uid) {
	// We're interested in TRIGGER_COUNT events in any
	// PERSISTENT_APP_CRASH_TRIGGER_WINDOW_MILLIS second period; apps crash pretty quickly
	// so we can keep a tight leash on them.
	super(uid, TRIGGER_COUNT, PERSISTENT_APP_CRASH_TRIGGER_WINDOW_MILLIS);
	}

	@Override public int getCount() { return count; }
	@Override public void setCount(int count) { this.count = count; }
	@Override public long getStart() { return start; }
	@Override public void setStart(long start) { this.start = start; }
	}

	private static int[] getAllUserIds() {		private static int[] getAllUserIds() {
	int[] userIds = { UserHandle.USER_SYSTEM };		int[] userIds = { UserHandle.USER_SYSTEM };
	try {		try {

services/core/java/com/android/server/am/AppErrors.java

+0 −25

Original line number	Original line	Diff line number	Diff line
	@@ -33,8 +33,6 @@ import android.content.ActivityNotFoundException;
	import android.content.Context;		import android.content.Context;
	import android.content.Intent;		import android.content.Intent;
	import android.content.pm.ApplicationInfo;		import android.content.pm.ApplicationInfo;
	import android.content.pm.ModuleInfo;
	import android.content.pm.PackageManager;
	import android.content.pm.VersionedPackage;		import android.content.pm.VersionedPackage;
	import android.net.Uri;		import android.net.Uri;
	import android.os.Binder;		import android.os.Binder;
	@@ -56,7 +54,6 @@ import com.android.internal.app.ProcessMap;
	import com.android.internal.logging.MetricsLogger;		import com.android.internal.logging.MetricsLogger;
	import com.android.internal.logging.nano.MetricsProto;		import com.android.internal.logging.nano.MetricsProto;
	import com.android.server.PackageWatchdog;		import com.android.server.PackageWatchdog;
	import com.android.server.RescueParty;
	import com.android.server.wm.WindowProcessController;		import com.android.server.wm.WindowProcessController;

	import java.io.FileDescriptor;		import java.io.FileDescriptor;
	@@ -423,28 +420,6 @@ class AppErrors {
	}		}

	if (r != null) {		if (r != null) {
	boolean isApexModule = false;
	try {
	for (String androidPackage : r.getPackageList()) {
	ModuleInfo moduleInfo = mContext.getPackageManager().getModuleInfo(
	androidPackage, /flags=/ 0);
	if (moduleInfo != null) {
	isApexModule = true;
	break;
	}
	}
	} catch (IllegalStateException \| PackageManager.NameNotFoundException e) {
	// Call to PackageManager#getModuleInfo() can result in NameNotFoundException or
	// IllegalStateException. In case they are thrown, there isn't much we can do
	// other than proceed with app crash handling.
	}

	if (r.isPersistent() \|\| isApexModule) {
	// If a persistent app or apex module is stuck in a crash loop, the device isn't
	// very usable, so we want to consider sending out a rescue party.
	RescueParty.noteAppCrash(mContext, r.uid);
	}

	mPackageWatchdog.onPackageFailure(r.getPackageListWithVersionCode(),		mPackageWatchdog.onPackageFailure(r.getPackageListWithVersionCode(),
	PackageWatchdog.FAILURE_REASON_APP_CRASH);		PackageWatchdog.FAILURE_REASON_APP_CRASH);
	}		}

services/java/com/android/server/SystemServer.java

+1 −0

Original line number	Original line	Diff line number	Diff line
	@@ -751,6 +751,7 @@ public final class SystemServer {
	// Now that we have the bare essentials of the OS up and running, take		// Now that we have the bare essentials of the OS up and running, take
	// note that we just booted, which might send out a rescue party if		// note that we just booted, which might send out a rescue party if
	// we're stuck in a runtime restart loop.		// we're stuck in a runtime restart loop.
			RescueParty.registerHealthObserver(mSystemContext);
	RescueParty.noteBoot(mSystemContext);		RescueParty.noteBoot(mSystemContext);

	// Manages LEDs and display backlight so we need it to bring up the display.		// Manages LEDs and display backlight so we need it to bring up the display.

services/tests/mockingservicestests/src/com/android/server/RescuePartyTest.java

+84 −45

Original line number	Original line	Diff line number	Diff line
	@@ -32,6 +32,7 @@ import static org.mockito.ArgumentMatchers.isNull;

	import android.content.ContentResolver;		import android.content.ContentResolver;
	import android.content.Context;		import android.content.Context;
			import android.content.pm.VersionedPackage;
	import android.os.RecoverySystem;		import android.os.RecoverySystem;
	import android.os.SystemProperties;		import android.os.SystemProperties;
	import android.os.UserHandle;		import android.os.UserHandle;
	@@ -39,6 +40,8 @@ import android.provider.DeviceConfig;
	import android.provider.Settings;		import android.provider.Settings;

	import com.android.dx.mockito.inline.extended.ExtendedMockito;		import com.android.dx.mockito.inline.extended.ExtendedMockito;
			import com.android.server.PackageWatchdog.PackageHealthObserverImpact;
			import com.android.server.RescueParty.RescuePartyObserver;
	import com.android.server.am.SettingsToPropertiesMapper;		import com.android.server.am.SettingsToPropertiesMapper;
	import com.android.server.utils.FlagNamespaceUtils;		import com.android.server.utils.FlagNamespaceUtils;

	@@ -57,13 +60,15 @@ import java.util.HashMap;
	* Test RescueParty.		* Test RescueParty.
	*/		*/
	public class RescuePartyTest {		public class RescuePartyTest {
	private static final int PERSISTENT_APP_UID = 12;
	private static final long CURRENT_NETWORK_TIME_MILLIS = 0L;		private static final long CURRENT_NETWORK_TIME_MILLIS = 0L;
	private static final String FAKE_NATIVE_NAMESPACE1 = "native1";		private static final String FAKE_NATIVE_NAMESPACE1 = "native1";
	private static final String FAKE_NATIVE_NAMESPACE2 = "native2";		private static final String FAKE_NATIVE_NAMESPACE2 = "native2";
	private static final String[] FAKE_RESET_NATIVE_NAMESPACES =		private static final String[] FAKE_RESET_NATIVE_NAMESPACES =
	{FAKE_NATIVE_NAMESPACE1, FAKE_NATIVE_NAMESPACE2};		{FAKE_NATIVE_NAMESPACE1, FAKE_NATIVE_NAMESPACE2};

			private static VersionedPackage sFailingPackage = new VersionedPackage("com.package.name", 1);
			private static final String PROP_DISABLE_RESCUE = "persist.sys.disable_rescue";

	private MockitoSession mSession;		private MockitoSession mSession;

	@Mock(answer = Answers.RETURNS_DEEP_STUBS)		@Mock(answer = Answers.RETURNS_DEEP_STUBS)
	@@ -182,25 +187,25 @@ public class RescuePartyTest {

	@Test		@Test
	public void testPersistentAppCrashDetectionWithExecutionForAllRescueLevels() {		public void testPersistentAppCrashDetectionWithExecutionForAllRescueLevels() {
	notePersistentAppCrash(RescueParty.TRIGGER_COUNT);		notePersistentAppCrash();

	verifySettingsResets(Settings.RESET_MODE_UNTRUSTED_DEFAULTS);		verifySettingsResets(Settings.RESET_MODE_UNTRUSTED_DEFAULTS);
	assertEquals(RescueParty.LEVEL_RESET_SETTINGS_UNTRUSTED_DEFAULTS,		assertEquals(RescueParty.LEVEL_RESET_SETTINGS_UNTRUSTED_DEFAULTS,
	SystemProperties.getInt(RescueParty.PROP_RESCUE_LEVEL, RescueParty.LEVEL_NONE));		SystemProperties.getInt(RescueParty.PROP_RESCUE_LEVEL, RescueParty.LEVEL_NONE));

	notePersistentAppCrash(RescueParty.TRIGGER_COUNT);		notePersistentAppCrash();

	verifySettingsResets(Settings.RESET_MODE_UNTRUSTED_CHANGES);		verifySettingsResets(Settings.RESET_MODE_UNTRUSTED_CHANGES);
	assertEquals(RescueParty.LEVEL_RESET_SETTINGS_UNTRUSTED_CHANGES,		assertEquals(RescueParty.LEVEL_RESET_SETTINGS_UNTRUSTED_CHANGES,
	SystemProperties.getInt(RescueParty.PROP_RESCUE_LEVEL, RescueParty.LEVEL_NONE));		SystemProperties.getInt(RescueParty.PROP_RESCUE_LEVEL, RescueParty.LEVEL_NONE));

	notePersistentAppCrash(RescueParty.TRIGGER_COUNT);		notePersistentAppCrash();

	verifySettingsResets(Settings.RESET_MODE_TRUSTED_DEFAULTS);		verifySettingsResets(Settings.RESET_MODE_TRUSTED_DEFAULTS);
	assertEquals(RescueParty.LEVEL_RESET_SETTINGS_TRUSTED_DEFAULTS,		assertEquals(RescueParty.LEVEL_RESET_SETTINGS_TRUSTED_DEFAULTS,
	SystemProperties.getInt(RescueParty.PROP_RESCUE_LEVEL, RescueParty.LEVEL_NONE));		SystemProperties.getInt(RescueParty.PROP_RESCUE_LEVEL, RescueParty.LEVEL_NONE));

	notePersistentAppCrash(RescueParty.TRIGGER_COUNT);		notePersistentAppCrash();

	verify(() -> RecoverySystem.rebootPromptAndWipeUserData(mMockContext, RescueParty.TAG));		verify(() -> RecoverySystem.rebootPromptAndWipeUserData(mMockContext, RescueParty.TAG));
	assertEquals(RescueParty.LEVEL_FACTORY_RESET,		assertEquals(RescueParty.LEVEL_FACTORY_RESET,
	@@ -220,20 +225,6 @@ public class RescuePartyTest {
	SystemProperties.getInt(RescueParty.PROP_RESCUE_LEVEL, RescueParty.LEVEL_NONE));		SystemProperties.getInt(RescueParty.PROP_RESCUE_LEVEL, RescueParty.LEVEL_NONE));
	}		}

	@Test
	public void testPersistentAppCrashDetectionWithWrongInterval() {
	notePersistentAppCrash(RescueParty.TRIGGER_COUNT - 1);

	// last persistent app crash is just outside of the boot loop detection window
	doReturn(CURRENT_NETWORK_TIME_MILLIS
	+ RescueParty.PERSISTENT_APP_CRASH_TRIGGER_WINDOW_MILLIS + 1)
	.when(() -> RescueParty.getElapsedRealtime());
	notePersistentAppCrash(/numTimes=/1);

	assertEquals(RescueParty.LEVEL_NONE,
	SystemProperties.getInt(RescueParty.PROP_RESCUE_LEVEL, RescueParty.LEVEL_NONE));
	}

	@Test		@Test
	public void testBootLoopDetectionWithProperInterval() {		public void testBootLoopDetectionWithProperInterval() {
	noteBoot(RescueParty.TRIGGER_COUNT - 1);		noteBoot(RescueParty.TRIGGER_COUNT - 1);
	@@ -248,21 +239,6 @@ public class RescuePartyTest {
	SystemProperties.getInt(RescueParty.PROP_RESCUE_LEVEL, RescueParty.LEVEL_NONE));		SystemProperties.getInt(RescueParty.PROP_RESCUE_LEVEL, RescueParty.LEVEL_NONE));
	}		}

	@Test
	public void testPersistentAppCrashDetectionWithProperInterval() {
	notePersistentAppCrash(RescueParty.TRIGGER_COUNT - 1);

	// last persistent app crash is just inside of the boot loop detection window
	doReturn(CURRENT_NETWORK_TIME_MILLIS
	+ RescueParty.PERSISTENT_APP_CRASH_TRIGGER_WINDOW_MILLIS)
	.when(() -> RescueParty.getElapsedRealtime());
	notePersistentAppCrash(/numTimes=/1);

	verifySettingsResets(Settings.RESET_MODE_UNTRUSTED_DEFAULTS);
	assertEquals(RescueParty.LEVEL_RESET_SETTINGS_UNTRUSTED_DEFAULTS,
	SystemProperties.getInt(RescueParty.PROP_RESCUE_LEVEL, RescueParty.LEVEL_NONE));
	}

	@Test		@Test
	public void testBootLoopDetectionWithWrongTriggerCount() {		public void testBootLoopDetectionWithWrongTriggerCount() {
	noteBoot(RescueParty.TRIGGER_COUNT - 1);		noteBoot(RescueParty.TRIGGER_COUNT - 1);
	@@ -270,13 +246,6 @@ public class RescuePartyTest {
	SystemProperties.getInt(RescueParty.PROP_RESCUE_LEVEL, RescueParty.LEVEL_NONE));		SystemProperties.getInt(RescueParty.PROP_RESCUE_LEVEL, RescueParty.LEVEL_NONE));
	}		}

	@Test
	public void testPersistentAppCrashDetectionWithWrongTriggerCount() {
	notePersistentAppCrash(RescueParty.TRIGGER_COUNT - 1);
	assertEquals(RescueParty.LEVEL_NONE,
	SystemProperties.getInt(RescueParty.PROP_RESCUE_LEVEL, RescueParty.LEVEL_NONE));
	}

	@Test		@Test
	public void testIsAttemptingFactoryReset() {		public void testIsAttemptingFactoryReset() {
	noteBoot(RescueParty.TRIGGER_COUNT * 4);		noteBoot(RescueParty.TRIGGER_COUNT * 4);
	@@ -319,6 +288,77 @@ public class RescuePartyTest {
	FAKE_NATIVE_NAMESPACE2, /makeDefault=/true));		FAKE_NATIVE_NAMESPACE2, /makeDefault=/true));
	}		}

			@Test
			public void testExplicitlyEnablingAndDisablingRescue() {
			SystemProperties.set(RescueParty.PROP_ENABLE_RESCUE, Boolean.toString(false));
			SystemProperties.set(PROP_DISABLE_RESCUE, Boolean.toString(true));
			assertEquals(RescuePartyObserver.getInstance(mMockContext).execute(sFailingPackage,
			PackageWatchdog.FAILURE_REASON_APP_NOT_RESPONDING), false);

			SystemProperties.set(RescueParty.PROP_ENABLE_RESCUE, Boolean.toString(true));
			assertTrue(RescuePartyObserver.getInstance(mMockContext).execute(sFailingPackage,
			PackageWatchdog.FAILURE_REASON_APP_NOT_RESPONDING));
			}

			@Test
			public void testHealthCheckLevels() {
			RescuePartyObserver observer = RescuePartyObserver.getInstance(mMockContext);

			// Ensure that no action is taken for cases where the failure reason is unknown
			SystemProperties.set(RescueParty.PROP_RESCUE_LEVEL, Integer.toString(
			RescueParty.LEVEL_FACTORY_RESET));
			assertEquals(observer.onHealthCheckFailed(null, PackageWatchdog.FAILURE_REASON_UNKNOWN),
			PackageHealthObserverImpact.USER_IMPACT_NONE);

			/*
			For the following cases, ensure that the returned user impact corresponds with the user
			impact of the next available rescue level, not the current one.
			*/
			SystemProperties.set(RescueParty.PROP_RESCUE_LEVEL, Integer.toString(
			RescueParty.LEVEL_NONE));
			assertEquals(observer.onHealthCheckFailed(null,
			PackageWatchdog.FAILURE_REASON_APP_NOT_RESPONDING),
			PackageHealthObserverImpact.USER_IMPACT_LOW);

			SystemProperties.set(RescueParty.PROP_RESCUE_LEVEL, Integer.toString(
			RescueParty.LEVEL_RESET_SETTINGS_UNTRUSTED_DEFAULTS));
			assertEquals(observer.onHealthCheckFailed(null,
			PackageWatchdog.FAILURE_REASON_APP_NOT_RESPONDING),
			PackageHealthObserverImpact.USER_IMPACT_LOW);


			SystemProperties.set(RescueParty.PROP_RESCUE_LEVEL, Integer.toString(
			RescueParty.LEVEL_RESET_SETTINGS_UNTRUSTED_CHANGES));
			assertEquals(observer.onHealthCheckFailed(null,
			PackageWatchdog.FAILURE_REASON_APP_NOT_RESPONDING),
			PackageHealthObserverImpact.USER_IMPACT_HIGH);


			SystemProperties.set(RescueParty.PROP_RESCUE_LEVEL, Integer.toString(
			RescueParty.LEVEL_RESET_SETTINGS_TRUSTED_DEFAULTS));
			assertEquals(observer.onHealthCheckFailed(null,
			PackageWatchdog.FAILURE_REASON_APP_NOT_RESPONDING),
			PackageHealthObserverImpact.USER_IMPACT_HIGH);


			SystemProperties.set(RescueParty.PROP_RESCUE_LEVEL, Integer.toString(
			RescueParty.LEVEL_FACTORY_RESET));
			assertEquals(observer.onHealthCheckFailed(null,
			PackageWatchdog.FAILURE_REASON_APP_NOT_RESPONDING),
			PackageHealthObserverImpact.USER_IMPACT_HIGH);
			}

			@Test
			public void testRescueLevelIncrementsWhenExecuted() {
			RescuePartyObserver observer = RescuePartyObserver.getInstance(mMockContext);
			SystemProperties.set(RescueParty.PROP_RESCUE_LEVEL, Integer.toString(
			RescueParty.LEVEL_NONE));
			observer.execute(sFailingPackage,
			PackageWatchdog.FAILURE_REASON_APP_CRASH);
			assertEquals(SystemProperties.getInt(RescueParty.PROP_RESCUE_LEVEL, -1),
			RescueParty.LEVEL_RESET_SETTINGS_UNTRUSTED_DEFAULTS);
			}

	private void verifySettingsResets(int resetMode) {		private void verifySettingsResets(int resetMode) {
	verify(() -> Settings.Global.resetToDefaultsAsUser(mMockContentResolver, null,		verify(() -> Settings.Global.resetToDefaultsAsUser(mMockContentResolver, null,
	resetMode, UserHandle.USER_SYSTEM));		resetMode, UserHandle.USER_SYSTEM));
	@@ -332,9 +372,8 @@ public class RescuePartyTest {
	}		}
	}		}

	private void notePersistentAppCrash(int numTimes) {		private void notePersistentAppCrash() {
	for (int i = 0; i < numTimes; i++) {		RescuePartyObserver.getInstance(mMockContext).execute(new VersionedPackage(
	RescueParty.noteAppCrash(mMockContext, PERSISTENT_APP_UID);		"com.package.name", 1), PackageWatchdog.FAILURE_REASON_UNKNOWN);
	}
	}		}
	}		}