Merge "Add tests for the system server watchdogs." into main (f2649e60) · Commits · e / os / android_frameworks_base

services/core/java/com/android/server/Watchdog.java

+58 −35

Original line number	Diff line number	Diff line
		@@ -69,6 +69,7 @@ import java.io.FileWriter;
		import java.io.IOException;
		import java.io.PrintWriter;
		import java.io.StringWriter;
		import java.time.Clock;
		import java.util.ArrayList;
		import java.util.Arrays;
		import java.util.Collections;
		@@ -105,10 +106,10 @@ public class Watchdog implements Dumpable {
		private static final int PRE_WATCHDOG_TIMEOUT_RATIO = 2;

		// These are temporally ordered: larger values as lateness increases
		private static final int COMPLETED = 0;
		private static final int WAITING = 1;
		private static final int WAITED_UNTIL_PRE_WATCHDOG = 2;
		private static final int OVERDUE = 3;
		static final int COMPLETED = 0;
		static final int WAITING = 1;
		static final int WAITED_UNTIL_PRE_WATCHDOG = 2;
		static final int OVERDUE = 3;

		// Track watchdog timeout history and break the crash loop if there is.
		private static final String TIMEOUT_HISTORY_FILE = "/data/system/watchdog-timeout-history.txt";
		@@ -243,10 +244,8 @@ public class Watchdog implements Dumpable {
		}
		}

		/**
		* Used for checking status of handle threads and scheduling monitor callbacks.
		*/
		public final class HandlerChecker implements Runnable {
		/** Used for checking status of handle threads and scheduling monitor callbacks. */
		public static class HandlerChecker implements Runnable {
		private final Handler mHandler;
		private final String mName;
		private final ArrayList<Monitor> mMonitors = new ArrayList<Monitor>();
		@@ -257,11 +256,19 @@ public class Watchdog implements Dumpable {
		private long mStartTimeMillis;
		private int mPauseCount;
		private long mPauseEndTimeMillis;
		private Clock mClock;
		private Object mLock;

		HandlerChecker(Handler handler, String name) {
		HandlerChecker(Handler handler, String name, Object lock, Clock clock) {
		mHandler = handler;
		mName = name;
		mLock = lock;
		mCompleted = true;
		mClock = clock;
		}

		HandlerChecker(Handler handler, String name, Object lock) {
		this(handler, name, lock, SystemClock.uptimeClock());
		}

		void addMonitorLocked(Monitor monitor) {
		@@ -284,11 +291,9 @@ public class Watchdog implements Dumpable {
		mMonitorQueue.clear();
		}

		long nowMillis = SystemClock.uptimeMillis();
		boolean isPaused = mPauseCount > 0
		\|\| (mPauseEndTimeMillis > 0 && mPauseEndTimeMillis < nowMillis);
		if ((mMonitors.size() == 0 && mHandler.getLooper().getQueue().isPolling())
		\|\| isPaused) {
		long nowMillis = mClock.millis();
		boolean isPaused = mPauseCount > 0 \|\| mPauseEndTimeMillis > nowMillis;
		if ((mMonitors.size() == 0 && isHandlerPolling()) \|\| isPaused) {
		// Don't schedule until after resume OR
		// If the target looper has recently been polling, then
		// there is no reason to enqueue our checker on it since that
		@@ -311,11 +316,15 @@ public class Watchdog implements Dumpable {
		mHandler.postAtFrontOfQueue(this);
		}

		boolean isHandlerPolling() {
		return mHandler.getLooper().getQueue().isPolling();
		}

		public int getCompletionStateLocked() {
		if (mCompleted) {
		return COMPLETED;
		} else {
		long latency = SystemClock.uptimeMillis() - mStartTimeMillis;
		long latency = mClock.millis() - mStartTimeMillis;
		if (latency < mWaitMaxMillis / PRE_WATCHDOG_TIMEOUT_RATIO) {
		return WAITING;
		} else if (latency < mWaitMaxMillis) {
		@@ -340,7 +349,7 @@ public class Watchdog implements Dumpable {
		} else {
		prefix = "Blocked in monitor " + mCurrentMonitor.getClass().getName();
		}
		long latencySeconds = (SystemClock.uptimeMillis() - mStartTimeMillis) / 1000;
		long latencySeconds = (mClock.millis() - mStartTimeMillis) / 1000;
		return prefix + " on " + mName + " (" + getThread().getName() + ")"
		+ " for " + latencySeconds + "s";
		}
		@@ -372,7 +381,7 @@ public class Watchdog implements Dumpable {
		* the given time.
		*/
		public void pauseForLocked(int pauseMillis, String reason) {
		mPauseEndTimeMillis = SystemClock.uptimeMillis() + pauseMillis;
		mPauseEndTimeMillis = mClock.millis() + pauseMillis;
		// Mark as completed, because there's a chance we called this after the watchog
		// thread loop called Object#wait after 'WAITED_UNTIL_PRE_WATCHDOG'. In that case we
		// want to ensure the next call to #getCompletionStateLocked for this checker returns
		@@ -404,6 +413,11 @@ public class Watchdog implements Dumpable {
		Slog.wtf(TAG, "Already resumed HandlerChecker: " + mName);
		}
		}

		@Override
		public String toString() {
		return "CheckerHandler for " + mName;
		}
		}

		final class RebootRequestReceiver extends BroadcastReceiver {
		@@ -453,31 +467,40 @@ public class Watchdog implements Dumpable {
		ServiceThread t = new ServiceThread("watchdog.monitor",
		android.os.Process.THREAD_PRIORITY_DEFAULT, true /allowIo/);
		t.start();
		mMonitorChecker = new HandlerChecker(new Handler(t.getLooper()), "monitor thread");
		mMonitorChecker = new HandlerChecker(new Handler(t.getLooper()), "monitor thread", mLock);
		mHandlerCheckers.add(withDefaultTimeout(mMonitorChecker));

		mHandlerCheckers.add(withDefaultTimeout(
		new HandlerChecker(FgThread.getHandler(), "foreground thread")));
		mHandlerCheckers.add(
		withDefaultTimeout(
		new HandlerChecker(FgThread.getHandler(), "foreground thread", mLock)));
		// Add checker for main thread. We only do a quick check since there
		// can be UI running on the thread.
		mHandlerCheckers.add(withDefaultTimeout(
		new HandlerChecker(new Handler(Looper.getMainLooper()), "main thread")));
		mHandlerCheckers.add(
		withDefaultTimeout(
		new HandlerChecker(
		new Handler(Looper.getMainLooper()), "main thread", mLock)));
		// Add checker for shared UI thread.
		mHandlerCheckers.add(withDefaultTimeout(
		new HandlerChecker(UiThread.getHandler(), "ui thread")));
		mHandlerCheckers.add(
		withDefaultTimeout(new HandlerChecker(UiThread.getHandler(), "ui thread", mLock)));
		// And also check IO thread.
		mHandlerCheckers.add(withDefaultTimeout(
		new HandlerChecker(IoThread.getHandler(), "i/o thread")));
		mHandlerCheckers.add(
		withDefaultTimeout(new HandlerChecker(IoThread.getHandler(), "i/o thread", mLock)));
		// And the display thread.
		mHandlerCheckers.add(withDefaultTimeout(
		new HandlerChecker(DisplayThread.getHandler(), "display thread")));
		mHandlerCheckers.add(
		withDefaultTimeout(
		new HandlerChecker(DisplayThread.getHandler(), "display thread", mLock)));
		// And the animation thread.
		mHandlerCheckers.add(withDefaultTimeout(
		new HandlerChecker(AnimationThread.getHandler(), "animation thread")));
		mHandlerCheckers.add(
		withDefaultTimeout(
		new HandlerChecker(
		AnimationThread.getHandler(), "animation thread", mLock)));
		// And the surface animation thread.
		mHandlerCheckers.add(withDefaultTimeout(
		new HandlerChecker(SurfaceAnimationThread.getHandler(),
		"surface animation thread")));
		mHandlerCheckers.add(
		withDefaultTimeout(
		new HandlerChecker(
		SurfaceAnimationThread.getHandler(),
		"surface animation thread",
		mLock)));
		// Initialize monitor for Binder threads.
		addMonitor(new BinderThreadMonitor());

		@@ -617,7 +640,7 @@ public class Watchdog implements Dumpable {
		public void addThread(Handler thread) {
		synchronized (mLock) {
		final String name = thread.getLooper().getThread().getName();
		mHandlerCheckers.add(withDefaultTimeout(new HandlerChecker(thread, name)));
		mHandlerCheckers.add(withDefaultTimeout(new HandlerChecker(thread, name, mLock)));
		}
		}

		@@ -625,7 +648,7 @@ public class Watchdog implements Dumpable {
		synchronized (mLock) {
		final String name = thread.getLooper().getThread().getName();
		mHandlerCheckers.add(
		withCustomTimeout(new HandlerChecker(thread, name), timeoutMillis));
		withCustomTimeout(new HandlerChecker(thread, name, mLock), timeoutMillis));
		}
		}

services/tests/servicestests/src/com/android/server/WatchdogTest.java

0 → 100644

+152 −0

Original line number	Diff line number	Diff line
		/*
		* Copyright (C) 2009 The Android Open Source Project
		*
		* Licensed under the Apache License, Version 2.0 (the "License");
		* you may not use this file except in compliance with the License.
		* You may obtain a copy of the License at
		*
		* http://www.apache.org/licenses/LICENSE-2.0
		*
		* Unless required by applicable law or agreed to in writing, software
		* distributed under the License is distributed on an "AS IS" BASIS,
		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
		* See the License for the specific language governing permissions and
		* limitations under the License.
		*/

		package com.android.server;

		import static org.junit.Assert.assertEquals;
		import static org.mockito.Mockito.mock;
		import static org.mockito.Mockito.verify;
		import static org.mockito.Mockito.verifyNoMoreInteractions;

		import android.os.Handler;
		import android.os.SimpleClock;

		import androidx.test.runner.AndroidJUnit4;

		import com.android.server.Watchdog.HandlerChecker;

		import org.junit.Before;
		import org.junit.Test;
		import org.junit.runner.RunWith;

		import java.time.ZoneOffset;

		/** Test class for {@link Watchdog}. */
		@RunWith(AndroidJUnit4.class)
		public class WatchdogTest {
		private static final int TIMEOUT_MS = 10;

		private TestClock mClock;
		private Handler mHandler;
		private HandlerChecker mChecker;

		@Before
		public void setUp() {
		mClock = new TestClock();
		mHandler = mock(Handler.class);
		mChecker =
		new HandlerChecker(mHandler, "monitor thread", new Object(), mClock) {
		@Override
		public boolean isHandlerPolling() {
		return false;
		}
		};
		}

		@Test
		public void checkerPausedUntilResume() {
		Watchdog.Monitor monitor = mock(Watchdog.Monitor.class);
		mChecker.addMonitorLocked(monitor);

		mChecker.pauseLocked("pausing");
		mChecker.scheduleCheckLocked(TIMEOUT_MS);
		verifyNoMoreInteractions(mHandler);
		assertEquals(Watchdog.COMPLETED, mChecker.getCompletionStateLocked());

		mChecker.resumeLocked("resuming");
		mChecker.scheduleCheckLocked(10);
		assertEquals(Watchdog.WAITING, mChecker.getCompletionStateLocked());
		}

		@Test
		public void checkerPausedUntilDeadline() {
		Watchdog.Monitor monitor = mock(Watchdog.Monitor.class);
		mChecker.addMonitorLocked(monitor);

		mChecker.pauseForLocked(10, "pausing");
		mChecker.scheduleCheckLocked(TIMEOUT_MS);
		verifyNoMoreInteractions(mHandler);
		assertEquals(Watchdog.COMPLETED, mChecker.getCompletionStateLocked());

		mClock.advanceBy(5);
		verifyNoMoreInteractions(mHandler);
		assertEquals(Watchdog.COMPLETED, mChecker.getCompletionStateLocked());

		// Above the 10s timeout. Watchdog should not be paused anymore.
		mClock.advanceBy(6);
		mChecker.scheduleCheckLocked(TIMEOUT_MS);
		assertEquals(Watchdog.WAITING, mChecker.getCompletionStateLocked());
		}

		@Test
		public void checkerPausedDuringScheduledRun() {
		Watchdog.Monitor monitor = mock(Watchdog.Monitor.class);
		mChecker.addMonitorLocked(monitor);

		mChecker.scheduleCheckLocked(TIMEOUT_MS);
		mClock.advanceBy(5);
		mChecker.pauseForLocked(10, "pausing");
		verifyNoMoreInteractions(mHandler);
		assertEquals(Watchdog.COMPLETED, mChecker.getCompletionStateLocked());

		// Above the 10s timeout. Watchdog should not be paused anymore.
		mClock.advanceBy(11);
		mChecker.scheduleCheckLocked(TIMEOUT_MS);
		assertEquals(Watchdog.WAITING, mChecker.getCompletionStateLocked());
		}

		@Test
		public void blockedThread() {
		mChecker.scheduleCheckLocked(TIMEOUT_MS);
		assertEquals(mChecker.getCompletionStateLocked(), Watchdog.WAITING);

		mClock.advanceBy(6);
		assertEquals(Watchdog.WAITED_UNTIL_PRE_WATCHDOG, mChecker.getCompletionStateLocked());

		// Above the 10s timeout.
		mClock.advanceBy(6);
		assertEquals(Watchdog.OVERDUE, mChecker.getCompletionStateLocked());
		}

		@Test
		public void checkNothingBlocked() {
		Watchdog.Monitor monitor = mock(Watchdog.Monitor.class);
		mChecker.addMonitorLocked(monitor);

		mChecker.scheduleCheckLocked(TIMEOUT_MS);
		// scheduleCheckLocked calls #postAtFrontOfQueue which will call mChecker.run().
		mChecker.run();
		assertEquals(Watchdog.COMPLETED, mChecker.getCompletionStateLocked());
		verify(monitor).monitor();
		}

		private static class TestClock extends SimpleClock {
		long mNowMillis = 1;

		TestClock() {
		super(ZoneOffset.UTC);
		}

		@Override
		public long millis() {
		return mNowMillis;
		}

		public void advanceBy(long millis) {
		mNowMillis += millis;
		}
		}
		}