Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit da0e9a9b authored by Terry Wang's avatar Terry Wang
Browse files

Add check for optimize after each mutation operation. 

Optimize is an important job for AppSearch to work properly.
It could collect garbage and release resources. Without it, the garbage
resource will last forever and the device's space will be fulled up by
zombie documents.

The algorithm to trigger optimize is:
1: Query garbage resource info after a batch of mutation operations.
2: Only trigger optimize if there are enough resource could be released.

The behavior exists in AppSearch Jetpack for a while and working properly
for our Jetpack dogfooders.

Bug: 175255572
Test: FrameworkOptimizeStrategyTest
Test: AppSearchConfigTest
Test: AppSearchImplTest
Change-Id: Ib7ae475cc035d1b69969df1e22ac409895e0e3fa
parent 733537f1

apex/appsearch/service/java/com/android/server/appsearch/AppSearchConfig.java

+72 −0
Original line number Diff line number Diff line
@@ -64,6 +64,12 @@ public final class AppSearchConfig implements AutoCloseable { 
    static final int DEFAULT_LIMIT_CONFIG_MAX_DOCUMENT_SIZE_BYTES = 512 * 1024; // 512KiB

    @VisibleForTesting

    static final int DEFAULT_LIMIT_CONFIG_MAX_DOCUMENT_COUNT = 20_000;

    @VisibleForTesting

    static final int DEFAULT_BYTES_OPTIMIZE_THRESHOLD = 1 * 1024 * 1024; // 1 MiB

    @VisibleForTesting

    static final int DEFAULT_TIME_OPTIMIZE_THRESHOLD_MILLIS = Integer.MAX_VALUE;

    @VisibleForTesting

    static final int DEFAULT_DOC_COUNT_OPTIMIZE_THRESHOLD = 10_000;



    /*

     * Keys for ALL the flags stored in DeviceConfig.
@@ -79,6 +85,9 @@ public final class AppSearchConfig implements AutoCloseable { 
            "limit_config_max_document_size_bytes";

    public static final String KEY_LIMIT_CONFIG_MAX_DOCUMENT_COUNT =

            "limit_config_max_document_docunt";

    public static final String KEY_BYTES_OPTIMIZE_THRESHOLD = "bytes_optimize_threshold";

    public static final String KEY_TIME_OPTIMIZE_THRESHOLD_MILLIS = "time_optimize_threshold";

    public static final String KEY_DOC_COUNT_OPTIMIZE_THRESHOLD = "doc_count_optimize_threshold";



    // Array contains all the corresponding keys for the cached values.

    private static final String[] KEYS_TO_ALL_CACHED_VALUES = {
@@ -88,6 +97,9 @@ public final class AppSearchConfig implements AutoCloseable { 
            KEY_SAMPLING_INTERVAL_FOR_PUT_DOCUMENT_STATS,

            KEY_LIMIT_CONFIG_MAX_DOCUMENT_SIZE_BYTES,

            KEY_LIMIT_CONFIG_MAX_DOCUMENT_COUNT,

            KEY_BYTES_OPTIMIZE_THRESHOLD,

            KEY_TIME_OPTIMIZE_THRESHOLD_MILLIS,

            KEY_DOC_COUNT_OPTIMIZE_THRESHOLD

    };



    // Lock needed for all the operations in this class.
@@ -251,6 +263,48 @@ public final class AppSearchConfig implements AutoCloseable { 
        }

    }



    /**

     * Returns the cached optimize byte size threshold.

     *

     * An AppSearch Optimize job will be triggered if the bytes size of garbage resource exceeds

     * this threshold.

     */

    int getCachedBytesOptimizeThreshold() {

        synchronized (mLock) {

            throwIfClosedLocked();

            return mBundleLocked.getInt(KEY_BYTES_OPTIMIZE_THRESHOLD,

                    DEFAULT_BYTES_OPTIMIZE_THRESHOLD);

        }

    }



    /**

     * Returns the cached optimize time interval threshold.

     *

     * An AppSearch Optimize job will be triggered if the time since last optimize job exceeds

     * this threshold.

     */

    int getCachedTimeOptimizeThresholdMs() {

        synchronized (mLock) {

            throwIfClosedLocked();

            return mBundleLocked.getInt(KEY_TIME_OPTIMIZE_THRESHOLD_MILLIS,

                    DEFAULT_TIME_OPTIMIZE_THRESHOLD_MILLIS);

        }

    }



    /**

     * Returns the cached optimize document count threshold threshold.

     *

     * An AppSearch Optimize job will be triggered if the number of document of garbage resource

     * exceeds this threshold.

     */

    int getCachedDocCountOptimizeThreshold() {

        synchronized (mLock) {

            throwIfClosedLocked();

            return mBundleLocked.getInt(KEY_DOC_COUNT_OPTIMIZE_THRESHOLD,

                    DEFAULT_DOC_COUNT_OPTIMIZE_THRESHOLD);

        }

    }



    @GuardedBy("mLock")

    private void throwIfClosedLocked() {

        if (mIsClosedLocked) {
@@ -307,6 +361,24 @@ public final class AppSearchConfig implements AutoCloseable { 
                            properties.getInt(key, DEFAULT_LIMIT_CONFIG_MAX_DOCUMENT_COUNT));

                }

                break;

            case KEY_BYTES_OPTIMIZE_THRESHOLD:

                synchronized (mLock) {

                    mBundleLocked.putInt(key, properties.getInt(key,

                            DEFAULT_BYTES_OPTIMIZE_THRESHOLD));

                }

                break;

            case KEY_TIME_OPTIMIZE_THRESHOLD_MILLIS:

                synchronized (mLock) {

                    mBundleLocked.putInt(key, properties.getInt(key,

                            DEFAULT_TIME_OPTIMIZE_THRESHOLD_MILLIS));

                }

                break;

            case KEY_DOC_COUNT_OPTIMIZE_THRESHOLD:

                synchronized (mLock) {

                    mBundleLocked.putInt(key, properties.getInt(key,

                            DEFAULT_DOC_COUNT_OPTIMIZE_THRESHOLD));

                }

                break;

            default:

                break;

        }

apex/appsearch/service/java/com/android/server/appsearch/AppSearchManagerService.java

+34 −0
Original line number Diff line number Diff line
@@ -364,6 +364,12 @@ public class AppSearchManagerService extends SystemService { 
                    ++operationSuccessCount;

                    invokeCallbackOnResult(callback,

                            AppSearchResult.newSuccessfulResult(setSchemaResponse.getBundle()));



                    // setSchema will sync the schemas in the request to AppSearch, any existing

                    // schemas which  is not included in the request will be delete if we force

                    // override incompatible schemas. And all documents of these types will be

                    // deleted as well. We should checkForOptimize for these deletion.

                    checkForOptimize(instance);

                } catch (Throwable t) {

                    ++operationFailureCount;

                    statusCode = throwableToFailedResult(t).getResultCode();
@@ -505,6 +511,10 @@ public class AppSearchManagerService extends SystemService { 
                    // Now that the batch has been written. Persist the newly written data.

                    instance.getAppSearchImpl().persistToDisk(PersistType.Code.LITE);

                    invokeCallbackOnResult(callback, resultBuilder.build());



                    // The existing documents with same ID will be deleted, so there may be some

                    // resources that could be released after optimize().

                    checkForOptimize(instance, /*mutateBatchSize=*/ documentBundles.size());

                } catch (Throwable t) {

                    ++operationFailureCount;

                    statusCode = throwableToFailedResult(t).getResultCode();
@@ -1023,6 +1033,8 @@ public class AppSearchManagerService extends SystemService { 
                    // Now that the batch has been written. Persist the newly written data.

                    instance.getAppSearchImpl().persistToDisk(PersistType.Code.LITE);

                    invokeCallbackOnResult(callback, resultBuilder.build());



                    checkForOptimize(instance, ids.size());

                } catch (Throwable t) {

                    ++operationFailureCount;

                    statusCode = throwableToFailedResult(t).getResultCode();
@@ -1092,6 +1104,8 @@ public class AppSearchManagerService extends SystemService { 
                    instance.getAppSearchImpl().persistToDisk(PersistType.Code.LITE);

                    ++operationSuccessCount;

                    invokeCallbackOnResult(callback, AppSearchResult.newSuccessfulResult(null));



                    checkForOptimize(instance);

                } catch (Throwable t) {

                    ++operationFailureCount;

                    statusCode = throwableToFailedResult(t).getResultCode();
@@ -1472,4 +1486,24 @@ public class AppSearchManagerService extends SystemService { 
            }

        }

    }



    private void checkForOptimize(AppSearchUserInstance instance, int mutateBatchSize) {

        EXECUTOR.execute(() -> {

            try {

                instance.getAppSearchImpl().checkForOptimize(mutateBatchSize);

            } catch (AppSearchException e) {

                Log.w(TAG, "Error occurred when check for optimize", e);

            }

        });

    }



    private void checkForOptimize(AppSearchUserInstance instance) {

        EXECUTOR.execute(() -> {

            try {

                instance.getAppSearchImpl().checkForOptimize();

            } catch (AppSearchException e) {

                Log.w(TAG, "Error occurred when check for optimize", e);

            }

        });

    }

}

apex/appsearch/service/java/com/android/server/appsearch/AppSearchUserInstanceManager.java

+1 −2
Original line number Diff line number Diff line
@@ -27,7 +27,6 @@ import android.util.Log; 


import com.android.internal.annotations.GuardedBy;

import com.android.server.appsearch.external.localstorage.AppSearchImpl;

import com.android.server.appsearch.external.localstorage.FrameworkOptimizeStrategy;

import com.android.server.appsearch.external.localstorage.stats.InitializeStats;

import com.android.server.appsearch.stats.PlatformLogger;

import com.android.server.appsearch.visibilitystore.VisibilityStoreImpl;
@@ -177,7 +176,7 @@ public final class AppSearchUserInstanceManager { 
                icingDir,

                new FrameworkLimitConfig(config),

                initStatsBuilder,

                new FrameworkOptimizeStrategy());

                new FrameworkOptimizeStrategy(config));



        long prepareVisibilityStoreLatencyStartMillis = SystemClock.elapsedRealtime();

        VisibilityStoreImpl visibilityStore =

apex/appsearch/service/java/com/android/server/appsearch/external/localstorage/FrameworkOptimizeStrategy.java→apex/appsearch/service/java/com/android/server/appsearch/FrameworkOptimizeStrategy.java

+16 −12
Original line number Diff line number Diff line 
/*

 * Copyright 2021 The Android Open Source Project

 * Copyright (C) 2021 The Android Open Source Project

 *

 * Licensed under the Apache License, Version 2.0 (the "License");

 * you may not use this file except in compliance with the License.
@@ -13,14 +13,17 @@ 
 * See the License for the specific language governing permissions and

 * limitations under the License.

 */

package com.android.server.appsearch.external.localstorage;

package com.android.server.appsearch;



import android.annotation.NonNull;



import com.android.internal.annotations.VisibleForTesting;

import com.android.server.appsearch.external.localstorage.AppSearchImpl;

import com.android.server.appsearch.external.localstorage.OptimizeStrategy;



import com.google.android.icing.proto.GetOptimizeInfoResultProto;



import java.util.Objects;



/**

 * An implementation of {@link OptimizeStrategy} will determine when to trigger {@link

 * AppSearchImpl#optimize()} in Jetpack environment.
@@ -28,17 +31,18 @@ import com.google.android.icing.proto.GetOptimizeInfoResultProto; 
 * @hide

 */

public class FrameworkOptimizeStrategy implements OptimizeStrategy {



    @VisibleForTesting static final int DOC_COUNT_OPTIMIZE_THRESHOLD = 100_000;

    @VisibleForTesting static final int BYTES_OPTIMIZE_THRESHOLD = 1 * 1024 * 1024 * 1024; // 1GB



    @VisibleForTesting

    static final long TIME_OPTIMIZE_THRESHOLD_MILLIS = 7 * 24 * 60 * 60 * 1000; // 1 week

    private final AppSearchConfig mAppSearchConfig;

    FrameworkOptimizeStrategy(@NonNull AppSearchConfig config) {

        mAppSearchConfig = Objects.requireNonNull(config);

    }



    @Override

    public boolean shouldOptimize(@NonNull GetOptimizeInfoResultProto optimizeInfo) {

        return optimizeInfo.getOptimizableDocs() >= DOC_COUNT_OPTIMIZE_THRESHOLD

                || optimizeInfo.getEstimatedOptimizableBytes() >= BYTES_OPTIMIZE_THRESHOLD

                || optimizeInfo.getTimeSinceLastOptimizeMs() >= TIME_OPTIMIZE_THRESHOLD_MILLIS;

        return optimizeInfo.getOptimizableDocs()

                    >= mAppSearchConfig.getCachedDocCountOptimizeThreshold()

                || optimizeInfo.getEstimatedOptimizableBytes()

                    >= mAppSearchConfig.getCachedBytesOptimizeThreshold()

                || optimizeInfo.getTimeSinceLastOptimizeMs()

                    >= mAppSearchConfig.getCachedTimeOptimizeThresholdMs();

    }

}

services/tests/mockingservicestests/src/com/android/server/appsearch/AppSearchConfigTest.java

+76 −5
Original line number Diff line number Diff line
@@ -54,6 +54,12 @@ public class AppSearchConfigTest { 
                AppSearchConfig.DEFAULT_LIMIT_CONFIG_MAX_DOCUMENT_SIZE_BYTES);

        assertThat(appSearchConfig.getCachedLimitConfigMaxDocumentCount()).isEqualTo(

                AppSearchConfig.DEFAULT_LIMIT_CONFIG_MAX_DOCUMENT_COUNT);

        assertThat(appSearchConfig.getCachedBytesOptimizeThreshold()).isEqualTo(

                AppSearchConfig.DEFAULT_BYTES_OPTIMIZE_THRESHOLD);

        assertThat(appSearchConfig.getCachedTimeOptimizeThresholdMs()).isEqualTo(

                AppSearchConfig.DEFAULT_TIME_OPTIMIZE_THRESHOLD_MILLIS);

        assertThat(appSearchConfig.getCachedDocCountOptimizeThreshold()).isEqualTo(

                AppSearchConfig.DEFAULT_DOC_COUNT_OPTIMIZE_THRESHOLD);

    }



    @Test
@@ -163,10 +169,8 @@ public class AppSearchConfigTest { 


    /**

     * Tests if we fall back to {@link AppSearchConfig#DEFAULT_SAMPLING_INTERVAL} if both default

     * sampling

     * interval and custom value are not set in DeviceConfig, and there is some other sampling

     * interval

     * set.

     * sampling interval and custom value are not set in DeviceConfig, and there is some other

     * sampling interval set.

     */

    @Test

    public void testFallbackToDefaultSamplingValue_useHardCodedDefault() {
@@ -269,7 +273,7 @@ public class AppSearchConfigTest { 
    }



    @Test

    public void testCustomizedValue() {

    public void testCustomizedValue_maxDocument() {

        DeviceConfig.setProperty(DeviceConfig.NAMESPACE_APPSEARCH,

                AppSearchConfig.KEY_LIMIT_CONFIG_MAX_DOCUMENT_SIZE_BYTES,

                Integer.toString(2001),
@@ -284,6 +288,64 @@ public class AppSearchConfigTest { 
        assertThat(appSearchConfig.getCachedLimitConfigMaxDocumentCount()).isEqualTo(2002);

    }



    @Test

    public void testCustomizedValue_optimizeThreshold() {

        DeviceConfig.setProperty(DeviceConfig.NAMESPACE_APPSEARCH,

                AppSearchConfig.KEY_BYTES_OPTIMIZE_THRESHOLD,

                Integer.toString(147147),

                false);

        DeviceConfig.setProperty(DeviceConfig.NAMESPACE_APPSEARCH,

                AppSearchConfig.KEY_TIME_OPTIMIZE_THRESHOLD_MILLIS,

                Integer.toString(258258),

                false);

        DeviceConfig.setProperty(DeviceConfig.NAMESPACE_APPSEARCH,

                AppSearchConfig.KEY_DOC_COUNT_OPTIMIZE_THRESHOLD,

                Integer.toString(369369),

                false);



        AppSearchConfig appSearchConfig = AppSearchConfig.create(DIRECT_EXECUTOR);



        assertThat(appSearchConfig.getCachedBytesOptimizeThreshold()).isEqualTo(147147);

        assertThat(appSearchConfig.getCachedTimeOptimizeThresholdMs()).isEqualTo(258258);

        assertThat(appSearchConfig.getCachedDocCountOptimizeThreshold()).isEqualTo(369369);

    }



    @Test

    public void testCustomizedValueOverride_optimizeThreshold() {

        DeviceConfig.setProperty(DeviceConfig.NAMESPACE_APPSEARCH,

                AppSearchConfig.KEY_BYTES_OPTIMIZE_THRESHOLD,

                Integer.toString(147147),

                false);

        DeviceConfig.setProperty(DeviceConfig.NAMESPACE_APPSEARCH,

                AppSearchConfig.KEY_TIME_OPTIMIZE_THRESHOLD_MILLIS,

                Integer.toString(258258),

                false);

        DeviceConfig.setProperty(DeviceConfig.NAMESPACE_APPSEARCH,

                AppSearchConfig.KEY_DOC_COUNT_OPTIMIZE_THRESHOLD,

                Integer.toString(369369),

                false);



        AppSearchConfig appSearchConfig = AppSearchConfig.create(DIRECT_EXECUTOR);



        // Override

        DeviceConfig.setProperty(DeviceConfig.NAMESPACE_APPSEARCH,

                AppSearchConfig.KEY_BYTES_OPTIMIZE_THRESHOLD,

                Integer.toString(741741),

                false);

        DeviceConfig.setProperty(DeviceConfig.NAMESPACE_APPSEARCH,

                AppSearchConfig.KEY_TIME_OPTIMIZE_THRESHOLD_MILLIS,

                Integer.toString(852852),

                false);

        DeviceConfig.setProperty(DeviceConfig.NAMESPACE_APPSEARCH,

                AppSearchConfig.KEY_DOC_COUNT_OPTIMIZE_THRESHOLD,

                Integer.toString(963963),

                false);



        assertThat(appSearchConfig.getCachedBytesOptimizeThreshold()).isEqualTo(741741);

        assertThat(appSearchConfig.getCachedTimeOptimizeThresholdMs()).isEqualTo(852852);

        assertThat(appSearchConfig.getCachedDocCountOptimizeThreshold()).isEqualTo(963963);

    }



    @Test

    public void testNotUsable_afterClose() {

        AppSearchConfig appSearchConfig = AppSearchConfig.create(DIRECT_EXECUTOR);
@@ -302,5 +364,14 @@ public class AppSearchConfigTest { 
        Assert.assertThrows("Trying to use a closed AppSearchConfig instance.",

                IllegalStateException.class,

                () -> appSearchConfig.getCachedSamplingIntervalForPutDocumentStats());

        Assert.assertThrows("Trying to use a closed AppSearchConfig instance.",

                IllegalStateException.class,

                () -> appSearchConfig.getCachedBytesOptimizeThreshold());

        Assert.assertThrows("Trying to use a closed AppSearchConfig instance.",

                IllegalStateException.class,

                () -> appSearchConfig.getCachedTimeOptimizeThresholdMs());

        Assert.assertThrows("Trying to use a closed AppSearchConfig instance.",

                IllegalStateException.class,

                () -> appSearchConfig.getCachedDocCountOptimizeThreshold());

    }

}