Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 9f41f464 authored by Gavin Corkery's avatar Gavin Corkery Committed by Android (Google) Code Review
Browse files

Merge changes from topic "pw-escalation"

* changes:
  Handle mitigation count logic in RescueParty
  Add per-package escalation logic
  Add getters/setters for ObserverInternal#packages
parents e875230f 68ec39e5
Loading
Loading
Loading
Loading
+114 −23
Original line number Diff line number Diff line
@@ -68,6 +68,7 @@ import java.util.Collections;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.NoSuchElementException;
import java.util.Set;
import java.util.concurrent.TimeUnit;

@@ -117,6 +118,9 @@ public class PackageWatchdog {
    static final int DEFAULT_TRIGGER_FAILURE_COUNT = 5;
    @VisibleForTesting
    static final long DEFAULT_OBSERVING_DURATION_MS = TimeUnit.DAYS.toMillis(2);
    // Sliding window for tracking how many mitigation calls were made for a package.
    @VisibleForTesting
    static final long DEFAULT_DEESCALATION_WINDOW_MS = TimeUnit.HOURS.toMillis(1);
    // Whether explicit health checks are enabled or not
    private static final boolean DEFAULT_EXPLICIT_HEALTH_CHECK_ENABLED = true;

@@ -388,6 +392,7 @@ public class PackageWatchdog {
                        // Observer that will receive failure for versionedPackage
                        PackageHealthObserver currentObserverToNotify = null;
                        int currentObserverImpact = Integer.MAX_VALUE;
                        MonitoredPackage currentMonitoredPackage = null;

                        // Find observer with least user impact
                        for (int oIndex = 0; oIndex < mAllObservers.size(); oIndex++) {
@@ -396,19 +401,33 @@ public class PackageWatchdog {
                            if (registeredObserver != null
                                    && observer.onPackageFailureLocked(
                                    versionedPackage.getPackageName())) {
                                MonitoredPackage p = observer.getMonitoredPackage(
                                        versionedPackage.getPackageName());
                                int mitigationCount = 1;
                                if (p != null) {
                                    mitigationCount = p.getMitigationCountLocked() + 1;
                                }
                                int impact = registeredObserver.onHealthCheckFailed(
                                        versionedPackage, failureReason);
                                        versionedPackage, failureReason, mitigationCount);
                                if (impact != PackageHealthObserverImpact.USER_IMPACT_NONE
                                        && impact < currentObserverImpact) {
                                    currentObserverToNotify = registeredObserver;
                                    currentObserverImpact = impact;
                                    currentMonitoredPackage = p;
                                }
                            }
                        }

                        // Execute action with least user impact
                        if (currentObserverToNotify != null) {
                            currentObserverToNotify.execute(versionedPackage, failureReason);
                            int mitigationCount = 1;
                            if (currentMonitoredPackage != null) {
                                currentMonitoredPackage.noteMitigationCallLocked();
                                mitigationCount =
                                        currentMonitoredPackage.getMitigationCountLocked();
                            }
                            currentObserverToNotify.execute(versionedPackage,
                                    failureReason, mitigationCount);
                        }
                    }
                }
@@ -429,7 +448,7 @@ public class PackageWatchdog {
            PackageHealthObserver registeredObserver = observer.registeredObserver;
            if (registeredObserver != null) {
                int impact = registeredObserver.onHealthCheckFailed(
                        failingPackage, failureReason);
                        failingPackage, failureReason, 1);
                if (impact != PackageHealthObserverImpact.USER_IMPACT_NONE
                        && impact < currentObserverImpact) {
                    currentObserverToNotify = registeredObserver;
@@ -438,7 +457,7 @@ public class PackageWatchdog {
            }
        }
        if (currentObserverToNotify != null) {
            currentObserverToNotify.execute(failingPackage,  failureReason);
            currentObserverToNotify.execute(failingPackage,  failureReason, 1);
        }
    }

@@ -559,6 +578,8 @@ public class PackageWatchdog {
         * @param versionedPackage the package that is failing. This may be null if a native
         *                          service is crashing.
         * @param failureReason   the type of failure that is occurring.
         * @param mitigationCount the number of times mitigation has been called for this package
         *                        (including this time).
         *
         *
         * @return any one of {@link PackageHealthObserverImpact} to express the impact
@@ -566,7 +587,8 @@ public class PackageWatchdog {
         */
        @PackageHealthObserverImpact int onHealthCheckFailed(
                @Nullable VersionedPackage versionedPackage,
                @FailureReasons int failureReason);
                @FailureReasons int failureReason,
                int mitigationCount);

        /**
         * Executes mitigation for {@link #onHealthCheckFailed}.
@@ -574,10 +596,12 @@ public class PackageWatchdog {
         * @param versionedPackage the package that is failing. This may be null if a native
         *                          service is crashing.
         * @param failureReason   the type of failure that is occurring.
         * @param mitigationCount the number of times mitigation has been called for this package
         *                        (including this time).
         * @return {@code true} if action was executed successfully, {@code false} otherwise
         */
        boolean execute(@Nullable VersionedPackage versionedPackage,
                @FailureReasons int failureReason);
                @FailureReasons int failureReason, int mitigationCount);


        /**
@@ -684,7 +708,7 @@ public class PackageWatchdog {
        synchronized (mLock) {
            for (int observerIdx = 0; observerIdx < mAllObservers.size(); observerIdx++) {
                ObserverInternal observer = mAllObservers.valueAt(observerIdx);
                MonitoredPackage monitoredPackage = observer.packages.get(packageName);
                MonitoredPackage monitoredPackage = observer.getMonitoredPackage(packageName);

                if (monitoredPackage != null) {
                    int oldState = monitoredPackage.getHealthCheckStateLocked();
@@ -713,7 +737,8 @@ public class PackageWatchdog {
            Slog.d(TAG, "Received supported packages " + supportedPackages);
            Iterator<ObserverInternal> oit = mAllObservers.values().iterator();
            while (oit.hasNext()) {
                Iterator<MonitoredPackage> pit = oit.next().packages.values().iterator();
                Iterator<MonitoredPackage> pit = oit.next().getMonitoredPackages()
                        .values().iterator();
                while (pit.hasNext()) {
                    MonitoredPackage monitoredPackage = pit.next();
                    String packageName = monitoredPackage.getName();
@@ -746,7 +771,7 @@ public class PackageWatchdog {
        while (oit.hasNext()) {
            ObserverInternal observer = oit.next();
            Iterator<MonitoredPackage> pit =
                    observer.packages.values().iterator();
                    observer.getMonitoredPackages().values().iterator();
            while (pit.hasNext()) {
                MonitoredPackage monitoredPackage = pit.next();
                String packageName = monitoredPackage.getName();
@@ -804,7 +829,8 @@ public class PackageWatchdog {
    private long getNextStateSyncMillisLocked() {
        long shortestDurationMs = Long.MAX_VALUE;
        for (int oIndex = 0; oIndex < mAllObservers.size(); oIndex++) {
            ArrayMap<String, MonitoredPackage> packages = mAllObservers.valueAt(oIndex).packages;
            ArrayMap<String, MonitoredPackage> packages = mAllObservers.valueAt(oIndex)
                    .getMonitoredPackages();
            for (int pIndex = 0; pIndex < packages.size(); pIndex++) {
                MonitoredPackage mp = packages.valueAt(pIndex);
                long duration = mp.getShortestScheduleDurationMsLocked();
@@ -838,7 +864,7 @@ public class PackageWatchdog {
            if (!failedPackages.isEmpty()) {
                onHealthCheckFailed(observer, failedPackages);
            }
            if (observer.packages.isEmpty() && (observer.registeredObserver == null
            if (observer.getMonitoredPackages().isEmpty() && (observer.registeredObserver == null
                    || !observer.registeredObserver.isPersistent())) {
                Slog.i(TAG, "Discarding observer " + observer.name + ". All packages expired");
                it.remove();
@@ -857,7 +883,7 @@ public class PackageWatchdog {
                        VersionedPackage versionedPkg = it.next().mPackage;
                        Slog.i(TAG, "Explicit health check failed for package " + versionedPkg);
                        registeredObserver.execute(versionedPkg,
                                PackageWatchdog.FAILURE_REASON_EXPLICIT_HEALTH_CHECK);
                                PackageWatchdog.FAILURE_REASON_EXPLICIT_HEALTH_CHECK, 1);
                    }
                }
            }
@@ -1054,7 +1080,7 @@ public class PackageWatchdog {
    private static class ObserverInternal {
        public final String name;
        @GuardedBy("mLock")
        public final ArrayMap<String, MonitoredPackage> packages = new ArrayMap<>();
        private final ArrayMap<String, MonitoredPackage> mPackages = new ArrayMap<>();
        @Nullable
        @GuardedBy("mLock")
        public PackageHealthObserver registeredObserver;
@@ -1073,8 +1099,8 @@ public class PackageWatchdog {
            try {
                out.startTag(null, TAG_OBSERVER);
                out.attribute(null, ATTR_NAME, name);
                for (int i = 0; i < packages.size(); i++) {
                    MonitoredPackage p = packages.valueAt(i);
                for (int i = 0; i < mPackages.size(); i++) {
                    MonitoredPackage p = mPackages.valueAt(i);
                    p.writeLocked(out);
                }
                out.endTag(null, TAG_OBSERVER);
@@ -1089,11 +1115,11 @@ public class PackageWatchdog {
        public void updatePackagesLocked(List<MonitoredPackage> packages) {
            for (int pIndex = 0; pIndex < packages.size(); pIndex++) {
                MonitoredPackage p = packages.get(pIndex);
                MonitoredPackage existingPackage = this.packages.get(p.getName());
                MonitoredPackage existingPackage = getMonitoredPackage(p.getName());
                if (existingPackage != null) {
                    existingPackage.updateHealthCheckDuration(p.mDurationMs);
                } else {
                    this.packages.put(p.getName(), p);
                    putMonitoredPackage(p);
                }
            }
        }
@@ -1111,7 +1137,7 @@ public class PackageWatchdog {
        @GuardedBy("mLock")
        private Set<MonitoredPackage> prunePackagesLocked(long elapsedMs) {
            Set<MonitoredPackage> failedPackages = new ArraySet<>();
            Iterator<MonitoredPackage> it = packages.values().iterator();
            Iterator<MonitoredPackage> it = mPackages.values().iterator();
            while (it.hasNext()) {
                MonitoredPackage p = it.next();
                int oldState = p.getHealthCheckStateLocked();
@@ -1134,18 +1160,52 @@ public class PackageWatchdog {
         */
        @GuardedBy("mLock")
        public boolean onPackageFailureLocked(String packageName) {
            if (packages.get(packageName) == null && registeredObserver.isPersistent()
            if (getMonitoredPackage(packageName) == null && registeredObserver.isPersistent()
                    && registeredObserver.mayObservePackage(packageName)) {
                packages.put(packageName, sPackageWatchdog.newMonitoredPackage(
                putMonitoredPackage(sPackageWatchdog.newMonitoredPackage(
                        packageName, DEFAULT_OBSERVING_DURATION_MS, false));
            }
            MonitoredPackage p = packages.get(packageName);
            MonitoredPackage p = getMonitoredPackage(packageName);
            if (p != null) {
                return p.onFailureLocked();
            }
            return false;
        }

        /**
         * Returns the map of packages monitored by this observer.
         *
         * @return a mapping of package names to {@link MonitoredPackage} objects.
         */
        @GuardedBy("mLock")
        public ArrayMap<String, MonitoredPackage> getMonitoredPackages() {
            return mPackages;
        }

        /**
         * Returns the {@link MonitoredPackage} associated with a given package name if the
         * package is being monitored by this observer.
         *
         * @param packageName: the name of the package.
         * @return the {@link MonitoredPackage} object associated with the package name if one
         *         exists, {@code null} otherwise.
         */
        @GuardedBy("mLock")
        @Nullable
        public MonitoredPackage getMonitoredPackage(String packageName) {
            return mPackages.get(packageName);
        }

        /**
         * Associates a {@link MonitoredPackage} with the observer.
         *
         * @param p: the {@link MonitoredPackage} to store.
         */
        @GuardedBy("mLock")
        public void putMonitoredPackage(MonitoredPackage p) {
            mPackages.put(p.getName(), p);
        }

        /**
         * Returns one ObserverInternal from the {@code parser} and advances its state.
         *
@@ -1201,8 +1261,8 @@ public class PackageWatchdog {
        public void dump(IndentingPrintWriter pw) {
            boolean isPersistent = registeredObserver != null && registeredObserver.isPersistent();
            pw.println("Persistent: " + isPersistent);
            for (String packageName : packages.keySet()) {
                MonitoredPackage p = packages.get(packageName);
            for (String packageName : mPackages.keySet()) {
                MonitoredPackage p = getMonitoredPackage(packageName);
                pw.println(packageName +  ": ");
                pw.increaseIndent();
                pw.println("# Failures: " + p.mFailureHistory.size());
@@ -1257,6 +1317,10 @@ public class PackageWatchdog {
        // Times when package failures happen sorted in ascending order
        @GuardedBy("mLock")
        private final LongArrayQueue mFailureHistory = new LongArrayQueue();
        // Times when an observer was called to mitigate this package's failure. Sorted in
        // ascending order.
        @GuardedBy("mLock")
        private final LongArrayQueue mMitigationCalls = new LongArrayQueue();
        // One of STATE_[ACTIVE|INACTIVE|PASSED|FAILED]. Updated on construction and after
        // methods that could change the health check state: handleElapsedTimeLocked and
        // tryPassHealthCheckLocked
@@ -1321,6 +1385,33 @@ public class PackageWatchdog {
            return failed;
        }

        /**
         * Notes the timestamp of a mitigation call into the observer.
         */
        @GuardedBy("mLock")
        public void noteMitigationCallLocked() {
            mMitigationCalls.addLast(mSystemClock.uptimeMillis());
        }

        /**
         * Prunes any mitigation calls outside of the de-escalation window, and returns the
         * number of calls that are in the window afterwards.
         *
         * @return the number of mitigation calls made in the de-escalation window.
         */
        @GuardedBy("mLock")
        public int getMitigationCountLocked() {
            try {
                final long now = mSystemClock.uptimeMillis();
                while (now - mMitigationCalls.peekFirst() > DEFAULT_DEESCALATION_WINDOW_MS) {
                    mMitigationCalls.removeFirst();
                }
            } catch (NoSuchElementException ignore) {
            }

            return mMitigationCalls.size();
        }

        /**
         * Sets the initial health check duration.
         *
+42 −25
Original line number Diff line number Diff line
@@ -77,6 +77,7 @@ public class RescueParty {
    static final String PROP_ENABLE_RESCUE = "persist.sys.enable_rescue";
    @VisibleForTesting
    static final String PROP_RESCUE_LEVEL = "sys.rescue_level";
    static final String PROP_ATTEMPTING_FACTORY_RESET = "sys.attempting_factory_reset";
    @VisibleForTesting
    static final int LEVEL_NONE = 0;
    @VisibleForTesting
@@ -155,7 +156,7 @@ public class RescueParty {
     * Check if we're currently attempting to reboot for a factory reset.
     */
    public static boolean isAttemptingFactoryReset() {
        return SystemProperties.getInt(PROP_RESCUE_LEVEL, LEVEL_NONE) == LEVEL_FACTORY_RESET;
        return SystemProperties.getBoolean(PROP_ATTEMPTING_FACTORY_RESET, false);
    }

    /**
@@ -230,14 +231,38 @@ public class RescueParty {
        }
    }

    private static int getMaxRescueLevel() {
        return SystemProperties.getBoolean(PROP_DISABLE_FACTORY_RESET_FLAG, false)
                ? LEVEL_RESET_SETTINGS_TRUSTED_DEFAULTS : LEVEL_FACTORY_RESET;
    }

    /**
     * Get the rescue level to perform if this is the n-th attempt at mitigating failure.
     *
     * @param mitigationCount: the mitigation attempt number (1 = first attempt etc.)
     * @return the rescue level for the n-th mitigation attempt.
     */
    private static int getRescueLevel(int mitigationCount) {
        if (mitigationCount == 1) {
            return LEVEL_RESET_SETTINGS_UNTRUSTED_DEFAULTS;
        } else if (mitigationCount == 2) {
            return LEVEL_RESET_SETTINGS_UNTRUSTED_CHANGES;
        } else if (mitigationCount == 3) {
            return LEVEL_RESET_SETTINGS_TRUSTED_DEFAULTS;
        } else if (mitigationCount >= 4) {
            return getMaxRescueLevel();
        } else {
            Slog.w(TAG, "Expected positive mitigation count, was " + mitigationCount);
            return LEVEL_NONE;
        }
    }

    /**
     * Get the next rescue level. This indicates the next level of mitigation that may be taken.
     */
    private static int getNextRescueLevel() {
        int maxRescueLevel = SystemProperties.getBoolean(PROP_DISABLE_FACTORY_RESET_FLAG, false)
                ? LEVEL_RESET_SETTINGS_TRUSTED_DEFAULTS : LEVEL_FACTORY_RESET;
        return MathUtils.constrain(SystemProperties.getInt(PROP_RESCUE_LEVEL, LEVEL_NONE) + 1,
                LEVEL_NONE, maxRescueLevel);
                LEVEL_NONE, getMaxRescueLevel());
    }

    /**
@@ -256,7 +281,11 @@ public class RescueParty {
    private static void executeRescueLevel(Context context, @Nullable String failedPackage) {
        final int level = SystemProperties.getInt(PROP_RESCUE_LEVEL, LEVEL_NONE);
        if (level == LEVEL_NONE) return;
        executeRescueLevel(context, failedPackage, level);
    }

    private static void executeRescueLevel(Context context, @Nullable String failedPackage,
            int level) {
        Slog.w(TAG, "Attempting rescue level " + levelToString(level));
        try {
            executeRescueLevelInternal(context, level, failedPackage);
@@ -284,6 +313,7 @@ public class RescueParty {
            case LEVEL_FACTORY_RESET:
                // Request the reboot from a separate thread to avoid deadlock on PackageWatchdog
                // when device shutting down.
                SystemProperties.set(PROP_ATTEMPTING_FACTORY_RESET, "true");
                Runnable runnable = new Runnable() {
                    @Override
                    public void run() {
@@ -320,15 +350,6 @@ public class RescueParty {
        }
    }

    private static int getPackageUid(Context context, String packageName) {
        try {
            return context.getPackageManager().getPackageUid(packageName, 0);
        } catch (PackageManager.NameNotFoundException e) {
            // Since UIDs are always >= 0, this value means the UID could not be determined.
            return -1;
        }
    }

    private static void resetAllSettings(Context context, int mode, @Nullable String failedPackage)
            throws Exception {
        // Try our best to reset all settings possible, and once finished
@@ -359,7 +380,7 @@ public class RescueParty {

    private static void resetDeviceConfig(Context context, int resetMode,
            @Nullable String failedPackage) {
        if (!shouldPerformScopedResets() || failedPackage == null) {
        if (!shouldPerformScopedResets(resetMode) || failedPackage == null) {
            resetAllAffectedNamespaces(context, resetMode);
        } else {
            performScopedReset(context, resetMode, failedPackage);
@@ -384,11 +405,8 @@ public class RescueParty {
        }
    }

    private static boolean shouldPerformScopedResets() {
        int rescueLevel = MathUtils.constrain(
                SystemProperties.getInt(PROP_RESCUE_LEVEL, LEVEL_NONE),
                LEVEL_NONE, LEVEL_FACTORY_RESET);
        return rescueLevel <= LEVEL_RESET_SETTINGS_UNTRUSTED_CHANGES;
    private static boolean shouldPerformScopedResets(int resetMode) {
        return resetMode <= Settings.RESET_MODE_UNTRUSTED_CHANGES;
    }

    private static void performScopedReset(Context context, int resetMode,
@@ -452,10 +470,10 @@ public class RescueParty {

        @Override
        public int onHealthCheckFailed(@Nullable VersionedPackage failedPackage,
                @FailureReasons int failureReason) {
                @FailureReasons int failureReason, int mitigationCount) {
            if (!isDisabled() && (failureReason == PackageWatchdog.FAILURE_REASON_APP_CRASH
                    || failureReason == PackageWatchdog.FAILURE_REASON_APP_NOT_RESPONDING)) {
                return mapRescueLevelToUserImpact(getNextRescueLevel());
                return mapRescueLevelToUserImpact(getRescueLevel(mitigationCount));
            } else {
                return PackageHealthObserverImpact.USER_IMPACT_NONE;
            }
@@ -463,16 +481,15 @@ public class RescueParty {

        @Override
        public boolean execute(@Nullable VersionedPackage failedPackage,
                @FailureReasons int failureReason) {
                @FailureReasons int failureReason, int mitigationCount) {
            if (isDisabled()) {
                return false;
            }
            if (failureReason == PackageWatchdog.FAILURE_REASON_APP_CRASH
                    || failureReason == PackageWatchdog.FAILURE_REASON_APP_NOT_RESPONDING) {
                int triggerUid = getPackageUid(mContext, failedPackage.getPackageName());
                incrementRescueLevel(triggerUid);
                final int level = getRescueLevel(mitigationCount);
                executeRescueLevel(mContext,
                        failedPackage == null ? null : failedPackage.getPackageName());
                        failedPackage == null ? null : failedPackage.getPackageName(), level);
                return true;
            } else {
                return false;
+2 −2
Original line number Diff line number Diff line
@@ -92,7 +92,7 @@ final class RollbackPackageHealthObserver implements PackageHealthObserver {

    @Override
    public int onHealthCheckFailed(@Nullable VersionedPackage failedPackage,
            @FailureReasons int failureReason) {
            @FailureReasons int failureReason, int mitigationCount) {
        // For native crashes, we will roll back any available rollbacks
        if (failureReason == PackageWatchdog.FAILURE_REASON_NATIVE_CRASH
                && !mContext.getSystemService(RollbackManager.class)
@@ -110,7 +110,7 @@ final class RollbackPackageHealthObserver implements PackageHealthObserver {

    @Override
    public boolean execute(@Nullable VersionedPackage failedPackage,
            @FailureReasons int rollbackReason) {
            @FailureReasons int rollbackReason, int mitigationCount) {
        if (rollbackReason == PackageWatchdog.FAILURE_REASON_NATIVE_CRASH) {
            mHandler.post(() -> rollbackAll());
            return true;
+20 −67

File changed.

Preview size limit exceeded, changes collapsed.

+46 −3

File changed.

Preview size limit exceeded, changes collapsed.