Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit fa012b35 authored by Dianne Hackborn's avatar Dianne Hackborn
Browse files

Improve watchdog error reporting. 

We now keep track of all the threads that are stopped, and
print stacks for all of them.  Also more threads are now adding
themselves to the watchdog.

Unfortunately the stack we get from threads is far less useful
than the stacks from the ANR report, because these don't include
any information about the lock the thread is blocked on and what
thread is holding that lock.  For example, here is a test of the
log output from causing a hang in the system process:

W/Watchdog( 5205): *** WATCHDOG KILLING SYSTEM PROCESS: com.android.server.am.ActivityManagerService, main thread
W/Watchdog( 5205): foreground thread stack trace:
W/Watchdog( 5205):     at com.android.server.am.ActivityManagerService.monitor(ActivityManagerService.java:14333)
W/Watchdog( 5205):     at com.android.server.Watchdog$HandlerChecker.run(Watchdog.java:142)
W/Watchdog( 5205):     at android.os.Handler.handleCallback(Handler.java:730)
W/Watchdog( 5205):     at android.os.Handler.dispatchMessage(Handler.java:92)
W/Watchdog( 5205):     at android.os.Looper.loop(Looper.java:137)
W/Watchdog( 5205):     at android.os.HandlerThread.run(HandlerThread.java:61)
W/Watchdog( 5205): main thread stack trace:
W/Watchdog( 5205):     at com.android.server.am.ActivityManagerService.broadcastIntent(ActivityManagerService.java:12252)
W/Watchdog( 5205):     at android.app.ContextImpl.sendBroadcastAsUser(ContextImpl.java:1158)
W/Watchdog( 5205):     at com.android.server.DropBoxManagerService$3.handleMessage(DropBoxManagerService.java:161)
W/Watchdog( 5205):     at android.os.Handler.dispatchMessage(Handler.java:99)
W/Watchdog( 5205):     at android.os.Looper.loop(Looper.java:137)
W/Watchdog( 5205):     at com.android.server.ServerThread.initAndLoop(SystemServer.java:1050)
W/Watchdog( 5205):     at com.android.server.SystemServer.init2(SystemServer.java:1125)
W/Watchdog( 5205):     at com.android.server.SystemServer.init1(Native Method)
W/Watchdog( 5205):     at com.android.server.SystemServer.main(SystemServer.java:1116)
W/Watchdog( 5205):     at java.lang.reflect.Method.invokeNative(Native Method)
W/Watchdog( 5205):     at java.lang.reflect.Method.invoke(Method.java:525)
W/Watchdog( 5205):     at com.android.internal.os.ZygoteInit$MethodAndArgsCaller.run(ZygoteInit.java:774)
W/Watchdog( 5205):     at com.android.internal.os.ZygoteInit.main(ZygoteInit.java:590)
W/Watchdog( 5205):     at dalvik.system.NativeStart.main(Native Method)
I/Process ( 5205): Sending signal. PID: 5205 SIG: 9

Change-Id: I8ff9892d8d072d8dc599a73de4bdb75e3b1a6e97
parent 5b48229b

services/java/com/android/server/Watchdog.java

+33 −12
Original line number Diff line number Diff line
@@ -120,6 +120,14 @@ public class Watchdog extends Thread { 
            return mCompleted;

        }



        public Thread getThread() {

            return mHandler.getLooper().getThread();

        }



        public String getName() {

            return mName;

        }



        public String describeBlockedStateLocked() {

            return mCurrentMonitor == null ? mName : mCurrentMonitor.getClass().getName();

        }
@@ -256,16 +264,24 @@ public class Watchdog extends Thread { 
        return true;

    }



    private String describeBlockedCheckersLocked() {

        StringBuilder builder = new StringBuilder(128);

    private ArrayList<HandlerChecker> getBlockedCheckersLocked() {

        ArrayList<HandlerChecker> checkers = new ArrayList<HandlerChecker>();

        for (int i=0; i<mHandlerCheckers.size(); i++) {

            HandlerChecker hc = mHandlerCheckers.get(i);

            if (!hc.isCompletedLocked()) {

                checkers.add(hc);

            }

        }

        return checkers;

    }



    private String describeCheckersLocked(ArrayList<HandlerChecker> checkers) {

        StringBuilder builder = new StringBuilder(128);

        for (int i=0; i<checkers.size(); i++) {

            if (builder.length() > 0) {

                builder.append(", ");

            }

                builder.append(hc.describeBlockedStateLocked());

            }

            builder.append(checkers.get(i).describeBlockedStateLocked());

        }

        return builder.toString();

    }
@@ -274,6 +290,7 @@ public class Watchdog extends Thread { 
    public void run() {

        boolean waitedHalf = false;

        while (true) {

            final ArrayList<HandlerChecker> blockedCheckers;

            final String name;

            final boolean allowRestart;

            synchronized (this) {
@@ -318,7 +335,8 @@ public class Watchdog extends Thread { 
                    continue;

                }



                name = describeBlockedCheckersLocked();

                blockedCheckers = getBlockedCheckersLocked();

                name = describeCheckersLocked(blockedCheckers);

                allowRestart = mAllowRestart;

            }
@@ -395,12 +413,15 @@ public class Watchdog extends Thread { 
                Slog.w(TAG, "Restart not allowed: Watchdog is *not* killing the system process");

            } else {

                Slog.w(TAG, "*** WATCHDOG KILLING SYSTEM PROCESS: " + name);

                Slog.w(TAG, "Main thread stack trace:");

                StackTraceElement[] stackTrace = Looper.getMainLooper().getThread().getStackTrace();

                for (int i=0; i<blockedCheckers.size(); i++) {

                    Slog.w(TAG, blockedCheckers.get(i).getName() + " stack trace:");

                    StackTraceElement[] stackTrace

                            = blockedCheckers.get(i).getThread().getStackTrace();

                    for (StackTraceElement element: stackTrace) {

                    Slog.w(TAG, "\tat " + element);

                        Slog.w(TAG, "    at " + element);

                    }

                }

                Slog.w(TAG, "<End of main thread stack trace>");

                Slog.w(TAG, "*** GOODBYE!");

                Process.killProcess(Process.myPid());

                System.exit(10);

            }

services/java/com/android/server/am/ActivityManagerService.java

+1 −0
Original line number Diff line number Diff line
@@ -1576,6 +1576,7 @@ public final class ActivityManagerService extends ActivityManagerNative 
            synchronized (this) {

                mService = m;

                mLooper = Looper.myLooper();

                Watchdog.getInstance().addThread(new Handler(mLooper), getName());

                notifyAll();

            }

































services/java/com/android/server/pm/PackageManagerService.java



+2
−0






























Original line number
Diff line number
Diff line








@@ -41,6 +41,7 @@ import com.android.server.DeviceStorageMonitorService;













import com.android.server.EventLogTags;















import com.android.server.IntentResolver;





























import com.android.server.Watchdog;















import org.xmlpull.v1.XmlPullParser;















import org.xmlpull.v1.XmlPullParserException;















import org.xmlpull.v1.XmlSerializer;










@@ -1093,6 +1094,7 @@ public class PackageManagerService extends IPackageManager.Stub {













        synchronized (mPackages) {















            mHandlerThread.start();















            mHandler = new PackageHandler(mHandlerThread.getLooper());













            Watchdog.getInstance().addThread(mHandler, mHandlerThread.getName());































            File dataDir = Environment.getDataDirectory();















            mAppDataDir = new File(dataDir, "data");

































services/java/com/android/server/power/PowerManagerService.java



+1
−0






























Original line number
Diff line number
Diff line








@@ -405,6 +405,7 @@ public final class PowerManagerService extends IPowerManager.Stub













        mHandler = new PowerManagerHandler(mHandlerThread.getLooper());































        Watchdog.getInstance().addMonitor(this);













        Watchdog.getInstance().addThread(mHandler, mHandlerThread.getName());































        // Forcibly turn the screen on at boot so that it is in a known power state.















        // We do this in init() rather than in the constructor because setting the