Merge "Add AnrAuxiliaryTaskExecutor" (b946fe30) · Commits · e / os / android_frameworks_base

services/core/java/com/android/server/SystemServerInitThreadPool.java

+6 −3

Original line number	Diff line number	Diff line
		@@ -32,6 +32,7 @@ import java.io.PrintWriter;
		import java.util.ArrayList;
		import java.util.List;
		import java.util.Objects;
		import java.util.concurrent.CompletableFuture;
		import java.util.concurrent.ExecutorService;
		import java.util.concurrent.Future;
		import java.util.concurrent.TimeUnit;
		@@ -192,10 +193,12 @@ public final class SystemServerInitThreadPool implements Dumpable {
		private static void dumpStackTraces() {
		final ArrayList<Integer> pids = new ArrayList<>();
		pids.add(Process.myPid());
		ActivityManagerService.dumpStackTraces(pids, /* processCpuTracker= */null,
		/* lastPids= */null, Watchdog.getInterestingNativePids(),
		ActivityManagerService.dumpStackTraces(pids,
		/* processCpuTracker= /null, / lastPids= */null,
		CompletableFuture.completedFuture(Watchdog.getInterestingNativePids()),
		/* logExceptionCreatingFile= /null, / subject= */null,
		/* criticalEventSection= /null, / latencyTracker= */null);
		/* criticalEventSection= */null, Runnable::run,
		/* latencyTracker= */null);
		}

		@Override

services/core/java/com/android/server/Watchdog.java

+5 −3

Original line number	Diff line number	Diff line
		@@ -49,7 +49,7 @@ import android.util.Dumpable;
		import android.util.EventLog;
		import android.util.Log;
		import android.util.Slog;
		import android.util.SparseArray;
		import android.util.SparseBooleanArray;

		import com.android.internal.os.BackgroundThread;
		import com.android.internal.os.ProcessCpuTracker;
		@@ -75,6 +75,7 @@ import java.util.HashSet;
		import java.util.List;
		import java.util.Optional;
		import java.util.UUID;
		import java.util.concurrent.CompletableFuture;
		import java.util.concurrent.TimeUnit;

		/**
		@@ -894,8 +895,9 @@ public class Watchdog implements Dumpable {
		ProcessCpuTracker processCpuTracker = new ProcessCpuTracker(false);
		StringWriter tracesFileException = new StringWriter();
		final File stack = ActivityManagerService.dumpStackTraces(
		pids, processCpuTracker, new SparseArray<>(), getInterestingNativePids(),
		tracesFileException, subject, criticalEvents, /* latencyTracker= */null);
		pids, processCpuTracker, new SparseBooleanArray(),
		CompletableFuture.completedFuture(getInterestingNativePids()), tracesFileException,
		subject, criticalEvents, Runnable::run, /* latencyTracker= */null);
		// Give some extra time to make sure the stack traces get written.
		// The system's been hanging for a whlie, another second or two won't hurt much.
		SystemClock.sleep(5000);

services/core/java/com/android/server/am/ActivityManagerService.java

+106 −47

Original line number	Diff line number	Diff line
		@@ -332,6 +332,7 @@ import android.util.Pair;
		import android.util.PrintWriterPrinter;
		import android.util.Slog;
		import android.util.SparseArray;
		import android.util.SparseBooleanArray;
		import android.util.SparseIntArray;
		import android.util.TimeUtils;
		import android.util.proto.ProtoOutputStream;
		@@ -467,11 +468,15 @@ import java.util.Map;
		import java.util.Objects;
		import java.util.Set;
		import java.util.UUID;
		import java.util.concurrent.CompletableFuture;
		import java.util.concurrent.CopyOnWriteArrayList;
		import java.util.concurrent.ExecutionException;
		import java.util.concurrent.Executor;
		import java.util.concurrent.Future;
		import java.util.concurrent.atomic.AtomicInteger;
		import java.util.concurrent.atomic.AtomicLong;
		import java.util.function.BiFunction;
		import java.util.function.Supplier;

		public class ActivityManagerService extends IActivityManager.Stub
		implements Watchdog.Monitor, BatteryStatsImpl.BatteryCallback, ActivityManagerGlobalLock {
		@@ -3427,14 +3432,15 @@ public class ActivityManagerService extends IActivityManager.Stub
		* @param lastPids of dalvik VM processes to dump stack traces for last
		* @param nativePids optional list of native pids to dump stack crawls
		* @param logExceptionCreatingFile optional writer to which we log errors creating the file
		* @param auxiliaryTaskExecutor executor to execute auxiliary tasks on
		* @param latencyTracker the latency tracker instance of the current ANR.
		*/
		public static File dumpStackTraces(ArrayList<Integer> firstPids,
		ProcessCpuTracker processCpuTracker, SparseArray<Boolean> lastPids,
		ArrayList<Integer> nativePids, StringWriter logExceptionCreatingFile,
		AnrLatencyTracker latencyTracker) {
		return dumpStackTraces(firstPids, processCpuTracker, lastPids, nativePids,
		logExceptionCreatingFile, null, null, null, latencyTracker);
		ProcessCpuTracker processCpuTracker, SparseBooleanArray lastPids,
		Future<ArrayList<Integer>> nativePidsFuture, StringWriter logExceptionCreatingFile,
		@NonNull Executor auxiliaryTaskExecutor, AnrLatencyTracker latencyTracker) {
		return dumpStackTraces(firstPids, processCpuTracker, lastPids, nativePidsFuture,
		logExceptionCreatingFile, null, null, null, auxiliaryTaskExecutor, latencyTracker);
		}

		/**
		@@ -3445,14 +3451,17 @@ public class ActivityManagerService extends IActivityManager.Stub
		* @param logExceptionCreatingFile optional writer to which we log errors creating the file
		* @param subject optional line related to the error
		* @param criticalEventSection optional lines containing recent critical events.
		* @param auxiliaryTaskExecutor executor to execute auxiliary tasks on
		* @param latencyTracker the latency tracker instance of the current ANR.
		*/
		public static File dumpStackTraces(ArrayList<Integer> firstPids,
		ProcessCpuTracker processCpuTracker, SparseArray<Boolean> lastPids,
		ArrayList<Integer> nativePids, StringWriter logExceptionCreatingFile,
		String subject, String criticalEventSection, AnrLatencyTracker latencyTracker) {
		return dumpStackTraces(firstPids, processCpuTracker, lastPids, nativePids,
		logExceptionCreatingFile, null, subject, criticalEventSection, latencyTracker);
		ProcessCpuTracker processCpuTracker, SparseBooleanArray lastPids,
		Future<ArrayList<Integer>> nativePidsFuture, StringWriter logExceptionCreatingFile,
		String subject, String criticalEventSection, @NonNull Executor auxiliaryTaskExecutor,
		AnrLatencyTracker latencyTracker) {
		return dumpStackTraces(firstPids, processCpuTracker, lastPids, nativePidsFuture,
		logExceptionCreatingFile, null, subject, criticalEventSection,
		auxiliaryTaskExecutor, latencyTracker);
		}

		/**
		@@ -3460,49 +3469,26 @@ public class ActivityManagerService extends IActivityManager.Stub
		* of the very first pid to be dumped.
		*/
		/* package */ static File dumpStackTraces(ArrayList<Integer> firstPids,
		ProcessCpuTracker processCpuTracker, SparseArray<Boolean> lastPids,
		ArrayList<Integer> nativePids, StringWriter logExceptionCreatingFile,
		ProcessCpuTracker processCpuTracker, SparseBooleanArray lastPids,
		Future<ArrayList<Integer>> nativePidsFuture, StringWriter logExceptionCreatingFile,
		AtomicLong firstPidEndOffset, String subject, String criticalEventSection,
		AnrLatencyTracker latencyTracker) {
		@NonNull Executor auxiliaryTaskExecutor, AnrLatencyTracker latencyTracker) {
		try {

		if (latencyTracker != null) {
		latencyTracker.dumpStackTracesStarted();
		}
		ArrayList<Integer> extraPids = null;

		Slog.i(TAG, "dumpStackTraces pids=" + lastPids + " nativepids=" + nativePids);
		Slog.i(TAG, "dumpStackTraces pids=" + lastPids);

		// Measure CPU usage as soon as we're called in order to get a realistic sampling
		// of the top users at the time of the request.
		if (processCpuTracker != null) {
		if (latencyTracker != null) {
		latencyTracker.processCpuTrackerMethodsCalled();
		}
		processCpuTracker.init();
		try {
		Thread.sleep(200);
		} catch (InterruptedException ignored) {
		}

		processCpuTracker.update();

		// We'll take the stack crawls of just the top apps using CPU.
		final int workingStatsNumber = processCpuTracker.countWorkingStats();
		extraPids = new ArrayList<>();
		for (int i = 0; i < workingStatsNumber && extraPids.size() < 5; i++) {
		ProcessCpuTracker.Stats stats = processCpuTracker.getWorkingStats(i);
		if (lastPids.indexOfKey(stats.pid) >= 0) {
		if (DEBUG_ANR) Slog.d(TAG, "Collecting stacks for extra pid " + stats.pid);

		extraPids.add(stats.pid);
		} else {
		Slog.i(TAG, "Skipping next CPU consuming process, not a java proc: "
		+ stats.pid);
		}
		}
		if (latencyTracker != null) {
		latencyTracker.processCpuTrackerMethodsReturned();
		}
		Supplier<ArrayList<Integer>> extraPidsSupplier = processCpuTracker != null
		? () -> getExtraPids(processCpuTracker, lastPids, latencyTracker) : null;
		Future<ArrayList<Integer>> extraPidsFuture = null;
		if (extraPidsSupplier != null) {
		extraPidsFuture =
		CompletableFuture.supplyAsync(extraPidsSupplier, auxiliaryTaskExecutor);
		}

		final File tracesDir = new File(ANR_TRACE_DIR);
		@@ -3536,7 +3522,8 @@ public class ActivityManagerService extends IActivityManager.Stub
		}

		long firstPidEndPos = dumpStackTraces(
		tracesFile.getAbsolutePath(), firstPids, nativePids, extraPids, latencyTracker);
		tracesFile.getAbsolutePath(), firstPids, nativePidsFuture,
		extraPidsFuture, latencyTracker);
		if (firstPidEndOffset != null) {
		firstPidEndOffset.set(firstPidEndPos);
		}
		@@ -3554,6 +3541,42 @@ public class ActivityManagerService extends IActivityManager.Stub
		private static SimpleDateFormat sAnrFileDateFormat;
		static final String ANR_FILE_PREFIX = "anr_";

		private static ArrayList<Integer> getExtraPids(ProcessCpuTracker processCpuTracker,
		SparseBooleanArray lastPids, AnrLatencyTracker latencyTracker) {
		if (latencyTracker != null) {
		latencyTracker.processCpuTrackerMethodsCalled();
		}
		ArrayList<Integer> extraPids = new ArrayList<>();
		processCpuTracker.init();
		try {
		Thread.sleep(200);
		} catch (InterruptedException ignored) {
		}

		processCpuTracker.update();

		// We'll take the stack crawls of just the top apps using CPU.
		final int workingStatsNumber = processCpuTracker.countWorkingStats();
		for (int i = 0; i < workingStatsNumber && extraPids.size() < 5; i++) {
		ProcessCpuTracker.Stats stats = processCpuTracker.getWorkingStats(i);
		if (lastPids.indexOfKey(stats.pid) >= 0) {
		if (DEBUG_ANR) {
		Slog.d(TAG, "Collecting stacks for extra pid " + stats.pid);
		}

		extraPids.add(stats.pid);
		} else {
		Slog.i(TAG,
		"Skipping next CPU consuming process, not a java proc: "
		+ stats.pid);
		}
		}
		if (latencyTracker != null) {
		latencyTracker.processCpuTrackerMethodsReturned();
		}
		return extraPids;
		}

		private static synchronized File createAnrDumpFile(File tracesDir) throws IOException {
		if (sAnrFileDateFormat == null) {
		sAnrFileDateFormat = new SimpleDateFormat("yyyy-MM-dd-HH-mm-ss-SSS");
		@@ -3660,8 +3683,8 @@ public class ActivityManagerService extends IActivityManager.Stub
		* @return The end offset of the trace of the very first PID
		*/
		public static long dumpStackTraces(String tracesFile,
		ArrayList<Integer> firstPids, ArrayList<Integer> nativePids,
		ArrayList<Integer> extraPids, AnrLatencyTracker latencyTracker) {
		ArrayList<Integer> firstPids, Future<ArrayList<Integer>> nativePidsFuture,
		Future<ArrayList<Integer>> extraPidsFuture, AnrLatencyTracker latencyTracker) {

		Slog.i(TAG, "Dumping to " + tracesFile);

		@@ -3682,6 +3705,7 @@ public class ActivityManagerService extends IActivityManager.Stub
		if (latencyTracker != null) {
		latencyTracker.dumpingFirstPidsStarted();
		}

		int num = firstPids.size();
		for (int i = 0; i < num; i++) {
		final int pid = firstPids.get(i);
		@@ -3723,6 +3747,10 @@ public class ActivityManagerService extends IActivityManager.Stub
		}

		// Next collect the stacks of the native pids
		ArrayList<Integer> nativePids = collectPids(nativePidsFuture, "native pids");

		Slog.i(TAG, "dumpStackTraces nativepids=" + nativePids);

		if (nativePids != null) {
		if (latencyTracker != null) {
		latencyTracker.dumpingNativePidsStarted();
		@@ -3758,6 +3786,19 @@ public class ActivityManagerService extends IActivityManager.Stub
		}

		// Lastly, dump stacks for all extra PIDs from the CPU tracker.
		ArrayList<Integer> extraPids = collectPids(extraPidsFuture, "extra pids");

		if (extraPidsFuture != null) {
		try {
		extraPids = extraPidsFuture.get();
		} catch (ExecutionException e) {
		Slog.w(TAG, "Failed to collect extra pids", e.getCause());
		} catch (InterruptedException e) {
		Slog.w(TAG, "Interrupted while collecting extra pids", e);
		}
		}
		Slog.i(TAG, "dumpStackTraces extraPids=" + extraPids);

		if (extraPids != null) {
		if (latencyTracker != null) {
		latencyTracker.dumpingExtraPidsStarted();
		@@ -3793,6 +3834,24 @@ public class ActivityManagerService extends IActivityManager.Stub
		return firstPidEnd;
		}

		private static ArrayList<Integer> collectPids(Future<ArrayList<Integer>> pidsFuture,
		String logName) {

		ArrayList<Integer> pids = null;

		if (pidsFuture == null) {
		return pids;
		}
		try {
		pids = pidsFuture.get();
		} catch (ExecutionException e) {
		Slog.w(TAG, "Failed to collect " + logName, e.getCause());
		} catch (InterruptedException e) {
		Slog.w(TAG, "Interrupted while collecting " + logName , e);
		}
		return pids;
		}

		@Override
		public boolean clearApplicationUserData(final String packageName, boolean keepState,
		final IPackageDataObserver observer, int userId) {

services/core/java/com/android/server/am/AnrHelper.java

+29 −3

Original line number	Diff line number	Diff line
		@@ -25,10 +25,15 @@ import android.os.Trace;
		import android.util.Slog;

		import com.android.internal.annotations.GuardedBy;
		import com.android.internal.annotations.VisibleForTesting;
		import com.android.internal.os.TimeoutRecord;
		import com.android.server.wm.WindowProcessController;

		import java.util.ArrayList;
		import java.util.concurrent.ExecutorService;
		import java.util.concurrent.LinkedBlockingQueue;
		import java.util.concurrent.ThreadFactory;
		import java.util.concurrent.ThreadPoolExecutor;
		import java.util.concurrent.TimeUnit;
		import java.util.concurrent.atomic.AtomicBoolean;

		@@ -51,6 +56,14 @@ class AnrHelper {
		*/
		private static final long CONSECUTIVE_ANR_TIME_MS = TimeUnit.MINUTES.toMillis(2);

		/**
		* The keep alive time for the threads in the helper threadpool executor
		*/
		private static final int AUX_THREAD_KEEP_ALIVE_SECOND = 10;

		private static final ThreadFactory sDefaultThreadFactory = r ->
		new Thread(r, "AnrAuxiliaryTaskExecutor");

		@GuardedBy("mAnrRecords")
		private final ArrayList<AnrRecord> mAnrRecords = new ArrayList<>();
		private final AtomicBoolean mRunning = new AtomicBoolean(false);
		@@ -66,8 +79,18 @@ class AnrHelper {
		@GuardedBy("mAnrRecords")
		private int mProcessingPid = -1;

		private final ExecutorService mAuxiliaryTaskExecutor;

		AnrHelper(final ActivityManagerService service) {
		this(service, new ThreadPoolExecutor(/* corePoolSize= / 0, / maximumPoolSize= */ 1,
		/* keepAliveTime= */ AUX_THREAD_KEEP_ALIVE_SECOND, TimeUnit.SECONDS,
		new LinkedBlockingQueue<>(), sDefaultThreadFactory));
		}

		@VisibleForTesting
		AnrHelper(ActivityManagerService service, ExecutorService auxExecutor) {
		mService = service;
		mAuxiliaryTaskExecutor = auxExecutor;
		}

		void appNotResponding(ProcessRecord anrProcess, TimeoutRecord timeoutRecord) {
		@@ -108,7 +131,8 @@ class AnrHelper {
		}
		timeoutRecord.mLatencyTracker.anrRecordPlacingOnQueueWithSize(mAnrRecords.size());
		mAnrRecords.add(new AnrRecord(anrProcess, activityShortComponentName, aInfo,
		parentShortComponentName, parentProcess, aboveSystem, timeoutRecord));
		parentShortComponentName, parentProcess, aboveSystem,
		mAuxiliaryTaskExecutor, timeoutRecord));
		}
		startAnrConsumerIfNeeded();
		} finally {
		@@ -204,11 +228,12 @@ class AnrHelper {
		final ApplicationInfo mAppInfo;
		final WindowProcessController mParentProcess;
		final boolean mAboveSystem;
		final ExecutorService mAuxiliaryTaskExecutor;
		final long mTimestamp = SystemClock.uptimeMillis();
		AnrRecord(ProcessRecord anrProcess, String activityShortComponentName,
		ApplicationInfo aInfo, String parentShortComponentName,
		WindowProcessController parentProcess, boolean aboveSystem,
		TimeoutRecord timeoutRecord) {
		ExecutorService auxiliaryTaskExecutor, TimeoutRecord timeoutRecord) {
		mApp = anrProcess;
		mPid = anrProcess.mPid;
		mActivityShortComponentName = activityShortComponentName;
		@@ -217,6 +242,7 @@ class AnrHelper {
		mAppInfo = aInfo;
		mParentProcess = parentProcess;
		mAboveSystem = aboveSystem;
		mAuxiliaryTaskExecutor = auxiliaryTaskExecutor;
		}

		void appNotResponding(boolean onlyDumpSelf) {
		@@ -224,7 +250,7 @@ class AnrHelper {
		mTimeoutRecord.mLatencyTracker.anrProcessingStarted();
		mApp.mErrorState.appNotResponding(mActivityShortComponentName, mAppInfo,
		mParentShortComponentName, mParentProcess, mAboveSystem,
		mTimeoutRecord, onlyDumpSelf);
		mTimeoutRecord, mAuxiliaryTaskExecutor, onlyDumpSelf);
		} finally {
		mTimeoutRecord.mLatencyTracker.anrProcessingEnded();
		}

services/core/java/com/android/server/am/ProcessErrorStateRecord.java

+59 −31

Original line number	Diff line number	Diff line
		@@ -44,7 +44,7 @@ import android.os.incremental.IncrementalMetrics;
		import android.provider.Settings;
		import android.util.EventLog;
		import android.util.Slog;
		import android.util.SparseArray;
		import android.util.SparseBooleanArray;

		import com.android.internal.annotations.CompositeRWLock;
		import com.android.internal.annotations.GuardedBy;
		@@ -62,8 +62,12 @@ import java.io.PrintWriter;
		import java.io.StringWriter;
		import java.util.ArrayList;
		import java.util.UUID;
		import java.util.concurrent.ExecutionException;
		import java.util.concurrent.ExecutorService;
		import java.util.concurrent.Future;
		import java.util.concurrent.atomic.AtomicLong;


		/**
		* The error state of the process, such as if it's crashing/ANR etc.
		*/
		@@ -259,12 +263,13 @@ class ProcessErrorStateRecord {
		void appNotResponding(String activityShortComponentName, ApplicationInfo aInfo,
		String parentShortComponentName, WindowProcessController parentProcess,
		boolean aboveSystem, TimeoutRecord timeoutRecord,
		boolean onlyDumpSelf) {
		ExecutorService auxiliaryTaskExecutor, boolean onlyDumpSelf) {
		String annotation = timeoutRecord.mReason;
		AnrLatencyTracker latencyTracker = timeoutRecord.mLatencyTracker;
		Future<?> updateCpuStatsNowFirstCall = null;

		ArrayList<Integer> firstPids = new ArrayList<>(5);
		SparseArray<Boolean> lastPids = new SparseArray<>(20);
		SparseBooleanArray lastPids = new SparseBooleanArray(20);

		mApp.getWindowProcessController().appEarlyNotResponding(annotation, () -> {
		latencyTracker.waitingOnAMSLockStarted();
		@@ -277,10 +282,15 @@ class ProcessErrorStateRecord {
		});

		long anrTime = SystemClock.uptimeMillis();

		if (isMonitorCpuUsage()) {
		updateCpuStatsNowFirstCall = auxiliaryTaskExecutor.submit(
		() -> {
		latencyTracker.updateCpuStatsNowCalled();
		mService.updateCpuStatsNow();
		latencyTracker.updateCpuStatsNowReturned();
		});

		}

		final boolean isSilentAnr;
		@@ -369,7 +379,7 @@ class ProcessErrorStateRecord {
		firstPids.add(myPid);
		if (DEBUG_ANR) Slog.i(TAG, "Adding likely IME: " + r);
		} else {
		lastPids.put(myPid, Boolean.TRUE);
		lastPids.put(myPid, true);
		if (DEBUG_ANR) Slog.i(TAG, "Adding ANR proc: " + r);
		}
		}
		@@ -432,8 +442,14 @@ class ProcessErrorStateRecord {
		report.append(currentPsiState);
		ProcessCpuTracker processCpuTracker = new ProcessCpuTracker(true);

		// We push the native pids collection task to the helper thread through
		// the Anr auxiliary task executor, and wait on it later after dumping the first pids
		Future<ArrayList<Integer>> nativePidsFuture =
		auxiliaryTaskExecutor.submit(
		() -> {
		latencyTracker.nativePidCollectionStarted();
		// don't dump native PIDs for background ANRs unless it is the process of interest
		// don't dump native PIDs for background ANRs unless
		// it is the process of interest
		String[] nativeProcs = null;
		if (isSilentAnr \|\| onlyDumpSelf) {
		for (int i = 0; i < NATIVE_STACKS_OF_INTEREST.length; i++) {
		@@ -446,7 +462,8 @@ class ProcessErrorStateRecord {
		nativeProcs = NATIVE_STACKS_OF_INTEREST;
		}

		int[] pids = nativeProcs == null ? null : Process.getPidsForCommands(nativeProcs);
		int[] pids = nativeProcs == null
		? null : Process.getPidsForCommands(nativeProcs);
		ArrayList<Integer> nativePids = null;

		if (pids != null) {
		@@ -456,6 +473,9 @@ class ProcessErrorStateRecord {
		}
		}
		latencyTracker.nativePidCollectionEnded();
		return nativePids;
		});

		// For background ANRs, don't pass the ProcessCpuTracker to
		// avoid spending 1/2 second collecting stats to rank lastPids.
		StringWriter tracesFileException = new StringWriter();
		@@ -463,10 +483,18 @@ class ProcessErrorStateRecord {
		final AtomicLong firstPidEndOffset = new AtomicLong(-1);
		File tracesFile = ActivityManagerService.dumpStackTraces(firstPids,
		isSilentAnr ? null : processCpuTracker, isSilentAnr ? null : lastPids,
		nativePids, tracesFileException, firstPidEndOffset, annotation, criticalEventLog,
		latencyTracker);
		nativePidsFuture, tracesFileException, firstPidEndOffset, annotation,
		criticalEventLog, auxiliaryTaskExecutor, latencyTracker);

		if (isMonitorCpuUsage()) {
		// Wait for the first call to finish
		try {
		updateCpuStatsNowFirstCall.get();
		} catch (ExecutionException e) {
		Slog.w(TAG, "Failed to update the CPU stats", e.getCause());
		} catch (InterruptedException e) {
		Slog.w(TAG, "Interrupted while updating the CPU stats", e);
		}
		mService.updateCpuStatsNow();
		mService.mAppProfiler.printCurrentCpuState(report, anrTime);
		info.append(processCpuTracker.printCurrentLoad());