Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 16522412 authored by Varun Shah's avatar Varun Shah
Browse files

Optimize service restart logic. 

If a non-persistent service is being restarted and it was killed due to
low memory previously, instead of restarting the service in one second,
we now restart it based on its last known oom_adj value. The backoff
logic of consistent restarts remains the same.

Additionally, when the service is due to restart, if the memory pressure
is still considered to be critical, the new logic will further delay the
service restart.

Initial test results: # of restarts dropped by 40-50% on average with
the optimized logic when the memory pressure was gradually increased.

Tested manually via memeater. Increased the memory pressure by 500 mb
at a time via the following command: adb shell memeater 500
On a 4gb device, no significant restarts were noticed when memory
pressure was increased by 2gb in total. When increased by 3gb in total,
services started to restart and the above results were observed via
the logged output over a 5 min period.

Bug: 79848281
Test: manually (steps listed above)
Change-Id: Idcd1300882321c6033e569736150a8bf7889678b
parent f6f46ce6

core/java/android/provider/Settings.java

+2 −0
Original line number Diff line number Diff line
@@ -11199,6 +11199,8 @@ public final class Settings { 
         * service_max_inactivity               (long)

         * service_bg_start_timeout             (long)

         * process_start_async                  (boolean)

         * use_mem_aware_service_restarts       (boolean)

         * service_restart_delay_duration       (long)

         * </pre>

         *

         * <p>

services/core/java/com/android/server/am/ActiveServices.java

+85 −26
Original line number Diff line number Diff line
@@ -82,6 +82,7 @@ import android.util.proto.ProtoOutputStream; 
import android.webkit.WebViewZygote;



import com.android.internal.R;

import com.android.internal.app.procstats.ProcessStats;

import com.android.internal.app.procstats.ServiceState;

import com.android.internal.messages.nano.SystemMessageProto;

import com.android.internal.notification.SystemNotificationChannels;
@@ -168,6 +169,8 @@ public final class ActiveServices { 
    /** Temporary list for holding the results of calls to {@link #collectPackageServicesLocked} */

    private ArrayList<ServiceRecord> mTmpCollectionResults = null;



    private int mNumServiceRestarts = 0;



    /**

     * For keeping ActiveForegroundApps retaining state while the screen is off.

     */
@@ -2327,6 +2330,8 @@ public final class ActiveServices { 
                r.restartDelay = mAm.mConstants.BOUND_SERVICE_CRASH_RESTART_DURATION

                        * (r.crashCount - 1);

            } else {

                mNumServiceRestarts++;

                if (!mAm.mConstants.FLAG_USE_MEM_AWARE_SERVICE_RESTARTS) {

                    // If it has been a "reasonably long time" since the service

                    // was started, then reset our restart duration back to

                    // the beginning, so we don't infinitely increase the duration
@@ -2341,27 +2346,42 @@ public final class ActiveServices { 
                            r.restartDelay = minDuration;

                        }

                    }

                } else {

                    // The service will be restarted based on the last known oom_adj value

                    // for the process when it was destroyed. A higher oom_adj value

                    // means that the service will be restarted quicker.

                    // If the service seems to keep crashing, the service restart counter will be

                    // incremented and the restart delay will have an exponential backoff.

                    final int lastOomAdj = r.app == null ? r.lastKnownOomAdj : r.app.setAdj;

                    if (r.restartCount > 1) {

                        r.restartDelay *= mAm.mConstants.SERVICE_RESTART_DURATION_FACTOR;

                        if (r.restartDelay < minDuration) {

                            r.restartDelay = minDuration;

                        }

                    } else {

                        if (lastOomAdj <= ProcessList.VISIBLE_APP_ADJ) {

                            r.restartDelay = 1 * 1000; // 1 second

                        } else if (lastOomAdj <= ProcessList.PERCEPTIBLE_APP_ADJ) {

                            r.restartDelay = 2 * 1000; // 2 seconds

                        } else if (lastOomAdj <= ProcessList.SERVICE_ADJ) {

                            r.restartDelay = 5 * 1000; // 5 seconds

                        } else if (lastOomAdj <= ProcessList.PREVIOUS_APP_ADJ) {

                            r.restartDelay = 10 * 1000; // 10 seconds

                        } else {

                            r.restartDelay = 20 * 1000; // 20 seconds

                        }

                    }

                    // If the last time the service restarted was more than a minute ago,

                    // reset the service restart count, otherwise increment by one.

                    r.restartCount = (now > (r.restartTime + resetTime)) ? 1 : (r.restartCount + 1);

                }

            }



            r.nextRestartTime = now + r.restartDelay;



            // Make sure that we don't end up restarting a bunch of services

            // all at the same time.

            boolean repeat;

            do {

                repeat = false;

                final long restartTimeBetween = mAm.mConstants.SERVICE_MIN_RESTART_TIME_BETWEEN;

                for (int i=mRestartingServices.size()-1; i>=0; i--) {

                    ServiceRecord r2 = mRestartingServices.get(i);

                    if (r2 != r && r.nextRestartTime >= (r2.nextRestartTime-restartTimeBetween)

                            && r.nextRestartTime < (r2.nextRestartTime+restartTimeBetween)) {

                        r.nextRestartTime = r2.nextRestartTime + restartTimeBetween;

                        r.restartDelay = r.nextRestartTime - now;

                        repeat = true;

                        break;

                    }

                }

            } while (repeat);

            ensureSpacedServiceRestarts(r, now);



        } else {

            // Persistent processes are immediately restarted, so there is no
@@ -2391,6 +2411,24 @@ public final class ActiveServices { 
        return canceled;

    }



    private void ensureSpacedServiceRestarts(ServiceRecord r, long now) {

        boolean repeat;

        do {

            repeat = false;

            final long restartTimeBetween = mAm.mConstants.SERVICE_MIN_RESTART_TIME_BETWEEN;

            for (int i = mRestartingServices.size() - 1; i >= 0; i--) {

                ServiceRecord r2 = mRestartingServices.get(i);

                if (r2 != r && r.nextRestartTime >= (r2.nextRestartTime - restartTimeBetween)

                        && r.nextRestartTime < (r2.nextRestartTime + restartTimeBetween)) {

                    r.nextRestartTime = r2.nextRestartTime + restartTimeBetween;

                    r.restartDelay = r.nextRestartTime - now;

                    repeat = true;

                    break;

                }

            }

        } while (repeat);

    }



    final void performServiceRestartLocked(ServiceRecord r) {

        if (!mRestartingServices.contains(r)) {

            return;
@@ -2465,6 +2503,26 @@ public final class ActiveServices { 
            return null;

        }



        // If the service is restarting, check the memory pressure - if it's low or critical, then

        // further delay the restart because it will most likely get killed again; if the pressure

        // is not low or critical, continue restarting.

        if (mAm.mConstants.FLAG_USE_MEM_AWARE_SERVICE_RESTARTS && whileRestarting) {

            final int memLevel = mAm.getMemoryTrimLevel();

            if (memLevel >= ProcessStats.ADJ_MEM_FACTOR_LOW) {

                final long now = SystemClock.uptimeMillis();

                // Delay the restart based on the current memory pressure

                // Default delay duration is 5 seconds

                r.restartDelay = memLevel * mAm.mConstants.SERVICE_RESTART_DELAY_DURATION;

                r.nextRestartTime = now + r.restartDelay;

                ensureSpacedServiceRestarts(r, now);

                mAm.mHandler.removeCallbacks(r.restarter);

                mAm.mHandler.postAtTime(r.restarter, r.nextRestartTime);

                Slog.w(TAG, "Delaying restart of crashed service " + r.shortInstanceName

                        + " in " + r.restartDelay + "ms due to continued memory pressure");

                return null;

            }

        }



        if (DEBUG_SERVICE) {

            Slog.v(TAG_SERVICE, "Bringing up " + r + " " + r.intent + " fg=" + r.fgRequired);

        }
@@ -3579,6 +3637,7 @@ public final class ActiveServices { 
                    || !mAm.mUserController.isUserRunning(sr.userId, 0)) {

                bringDownServiceLocked(sr);

            } else {

                sr.lastKnownOomAdj = app.setAdj;

                boolean canceled = scheduleServiceRestartLocked(sr, true);



                // Should the service remain running?  Note that in the

services/core/java/com/android/server/am/ActivityManagerConstants.java

+18 −0
Original line number Diff line number Diff line
@@ -76,6 +76,8 @@ final class ActivityManagerConstants extends ContentObserver { 
    static final String KEY_COMPACT_THROTTLE_2 = "compact_throttle_2";

    static final String KEY_COMPACT_THROTTLE_3 = "compact_throttle_3";

    static final String KEY_COMPACT_THROTTLE_4 = "compact_throttle_4";

    static final String KEY_USE_MEM_AWARE_SERVICE_RESTARTS = "use_mem_aware_service_restarts";

    static final String KEY_SERVICE_RESTART_DELAY_DURATION = "service_restart_delay_duration";



    private static final int DEFAULT_MAX_CACHED_PROCESSES = 32;

    private static final long DEFAULT_BACKGROUND_SETTLE_TIME = 60*1000;
@@ -113,6 +115,8 @@ final class ActivityManagerConstants extends ContentObserver { 
    public static final long DEFAULT_COMPACT_THROTTLE_2 = 10000;

    public static final long DEFAULT_COMPACT_THROTTLE_3 = 500;

    public static final long DEFAULT_COMPACT_THROTTLE_4 = 10000;

    private static final boolean DEFAULT_USE_MEM_AWARE_SERVICE_RESTARTS = true;

    private static final long DEFAULT_SERVICE_RESTART_DELAY_DURATION = 5 * 1000;



    // Maximum number of cached processes we will allow.

    public int MAX_CACHED_PROCESSES = DEFAULT_MAX_CACHED_PROCESSES;
@@ -249,6 +253,12 @@ final class ActivityManagerConstants extends ContentObserver { 
    // How long we'll skip second compactAppFull after first compactAppFull

    public long COMPACT_THROTTLE_4 = DEFAULT_COMPACT_THROTTLE_4;



    // Use memory aware optimized logic to restart services

    public boolean FLAG_USE_MEM_AWARE_SERVICE_RESTARTS = DEFAULT_USE_MEM_AWARE_SERVICE_RESTARTS;



    // How long a service restart will be delayed by if there is memory pressure

    public long SERVICE_RESTART_DELAY_DURATION = DEFAULT_SERVICE_RESTART_DELAY_DURATION;



    // Indicates whether the activity starts logging is enabled.

    // Controlled by Settings.Global.ACTIVITY_STARTS_LOGGING_ENABLED

    volatile boolean mFlagActivityStartsLoggingEnabled;
@@ -413,6 +423,10 @@ final class ActivityManagerConstants extends ContentObserver { 
            COMPACT_THROTTLE_2 = mParser.getLong(KEY_COMPACT_THROTTLE_2, DEFAULT_COMPACT_THROTTLE_2);

            COMPACT_THROTTLE_3 = mParser.getLong(KEY_COMPACT_THROTTLE_3, DEFAULT_COMPACT_THROTTLE_3);

            COMPACT_THROTTLE_4 = mParser.getLong(KEY_COMPACT_THROTTLE_4, DEFAULT_COMPACT_THROTTLE_4);

            FLAG_USE_MEM_AWARE_SERVICE_RESTARTS = mParser.getBoolean(

                    KEY_USE_MEM_AWARE_SERVICE_RESTARTS, DEFAULT_USE_MEM_AWARE_SERVICE_RESTARTS);

            SERVICE_RESTART_DELAY_DURATION = mParser.getLong(

                    KEY_SERVICE_RESTART_DELAY_DURATION, DEFAULT_SERVICE_RESTART_DELAY_DURATION);



            updateMaxCachedProcesses();

        }
@@ -505,6 +519,10 @@ final class ActivityManagerConstants extends ContentObserver { 
        pw.println(TOP_TO_FGS_GRACE_DURATION);

        pw.print("  "); pw.print(KEY_USE_COMPACTION); pw.print("=");

        pw.println(USE_COMPACTION);

        pw.print("  "); pw.print(KEY_USE_MEM_AWARE_SERVICE_RESTARTS); pw.print("=");

        pw.println(FLAG_USE_MEM_AWARE_SERVICE_RESTARTS);

        pw.print("  "); pw.print(KEY_SERVICE_RESTART_DELAY_DURATION); pw.print("=");

        pw.println(SERVICE_RESTART_DELAY_DURATION);



        pw.println();

        if (mOverrideMaxCachedProcesses >= 0) {

services/core/java/com/android/server/am/ServiceRecord.java

+8 −7
Original line number Diff line number Diff line
@@ -16,10 +16,8 @@ 


package com.android.server.am;



import com.android.internal.app.procstats.ServiceState;

import com.android.internal.os.BatteryStatsImpl;

import com.android.server.LocalServices;

import com.android.server.notification.NotificationManagerInternal;

import static com.android.server.am.ActivityManagerDebugConfig.TAG_AM;

import static com.android.server.am.ActivityManagerDebugConfig.TAG_WITH_CLASS_NAME;



import android.app.INotificationManager;

import android.app.Notification;
@@ -44,6 +42,11 @@ import android.util.Slog; 
import android.util.TimeUtils;

import android.util.proto.ProtoOutputStream;

import android.util.proto.ProtoUtils;



import com.android.internal.app.procstats.ServiceState;

import com.android.internal.os.BatteryStatsImpl;

import com.android.server.LocalServices;

import com.android.server.notification.NotificationManagerInternal;

import com.android.server.uri.NeededUriGrants;

import com.android.server.uri.UriPermissionOwner;
@@ -52,9 +55,6 @@ import java.util.ArrayList; 
import java.util.List;

import java.util.Objects;



import static com.android.server.am.ActivityManagerDebugConfig.TAG_AM;

import static com.android.server.am.ActivityManagerDebugConfig.TAG_WITH_CLASS_NAME;



/**

 * A running application service.

 */
@@ -114,6 +114,7 @@ final class ServiceRecord extends Binder implements ComponentName.WithComponentN 
    long executingStart;    // start time of last execute request.

    boolean createdFromFg;  // was this service last created due to a foreground process call?

    int crashCount;         // number of times proc has crashed with service running

    int lastKnownOomAdj;    // last known OOM adjustment for process (used if ProcessRecord is null)

    int totalRestartCount;  // number of times we have had to restart.

    int restartCount;       // number of restarts performed in a row.

    long restartDelay;      // delay until next restart attempt.