Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 41061847 authored by Alexander Dorokhine's avatar Alexander Dorokhine Committed by Xiaoyu Jin
Browse files

Log stats for Optimize 

OptimizeStats is crucial for us to tune our configuration for
optimization, which might affect the health of AppSearch and system
overall. Details can be found in Ib7ae475cc035d1b69969df1e22ac409895e0e3fa.

Also add sampling parameters in AppSearchConfig for:
1) initializeStats
2) searchStats
3) globalSearchStats
4) optimizeStats

Bug: 173532925
Bug: 175255572
Test: AppSearchStatsTest, FrameworksMockingServicesTests:AppSearchConfigTest, CtsAppSearchTestCases
Ignore-AOSP-First: AppSearch is not available on AOSP yet
Change-Id: Id61704407bdb40707cb8fa46b556024aea9f9083
parent 09923d5e

apex/appsearch/service/java/com/android/server/appsearch/AppSearchConfig.java

+68 −0
Original line number Diff line number Diff line
@@ -81,6 +81,14 @@ public final class AppSearchConfig implements AutoCloseable { 
            "sampling_interval_for_batch_call_stats";

    public static final String KEY_SAMPLING_INTERVAL_FOR_PUT_DOCUMENT_STATS =

            "sampling_interval_for_put_document_stats";

    public static final String KEY_SAMPLING_INTERVAL_FOR_INITIALIZE_STATS =

            "sampling_interval_for_initialize_stats";

    public static final String KEY_SAMPLING_INTERVAL_FOR_SEARCH_STATS =

            "sampling_interval_for_search_stats";

    public static final String KEY_SAMPLING_INTERVAL_FOR_GLOBAL_SEARCH_STATS =

            "sampling_interval_for_global_search_stats";

    public static final String KEY_SAMPLING_INTERVAL_FOR_OPTIMIZE_STATS =

            "sampling_interval_for_optimize_stats";

    public static final String KEY_LIMIT_CONFIG_MAX_DOCUMENT_SIZE_BYTES =

            "limit_config_max_document_size_bytes";

    public static final String KEY_LIMIT_CONFIG_MAX_DOCUMENT_COUNT =
@@ -95,6 +103,10 @@ public final class AppSearchConfig implements AutoCloseable { 
            KEY_SAMPLING_INTERVAL_DEFAULT,

            KEY_SAMPLING_INTERVAL_FOR_BATCH_CALL_STATS,

            KEY_SAMPLING_INTERVAL_FOR_PUT_DOCUMENT_STATS,

            KEY_SAMPLING_INTERVAL_FOR_INITIALIZE_STATS,

            KEY_SAMPLING_INTERVAL_FOR_SEARCH_STATS,

            KEY_SAMPLING_INTERVAL_FOR_GLOBAL_SEARCH_STATS,

            KEY_SAMPLING_INTERVAL_FOR_OPTIMIZE_STATS,

            KEY_LIMIT_CONFIG_MAX_DOCUMENT_SIZE_BYTES,

            KEY_LIMIT_CONFIG_MAX_DOCUMENT_COUNT,

            KEY_BYTES_OPTIMIZE_THRESHOLD,
@@ -245,6 +257,58 @@ public final class AppSearchConfig implements AutoCloseable { 
        }

    }



    /**

     * Returns cached value for sampling interval for initialize.

     *

     * <p>For example, sampling_interval=10 means that one out of every 10 stats was logged.

     */

    public int getCachedSamplingIntervalForInitializeStats() {

        synchronized (mLock) {

            throwIfClosedLocked();

            return mBundleLocked.getInt(KEY_SAMPLING_INTERVAL_FOR_INITIALIZE_STATS,

                    getCachedSamplingIntervalDefault());

        }

    }



    /**

     * Returns cached value for sampling interval for search.

     *

     * <p>For example, sampling_interval=10 means that one out of every 10 stats was logged.

     */

    public int getCachedSamplingIntervalForSearchStats() {

        synchronized (mLock) {

            throwIfClosedLocked();

            return mBundleLocked.getInt(KEY_SAMPLING_INTERVAL_FOR_SEARCH_STATS,

                    getCachedSamplingIntervalDefault());

        }

    }



    /**

     * Returns cached value for sampling interval for globalSearch.

     *

     * <p>For example, sampling_interval=10 means that one out of every 10 stats was logged.

     */

    public int getCachedSamplingIntervalForGlobalSearchStats() {

        synchronized (mLock) {

            throwIfClosedLocked();

            return mBundleLocked.getInt(KEY_SAMPLING_INTERVAL_FOR_GLOBAL_SEARCH_STATS,

                    getCachedSamplingIntervalDefault());

        }

    }



    /**

     * Returns cached value for sampling interval for optimize.

     *

     * <p>For example, sampling_interval=10 means that one out of every 10 stats was logged.

     */

    public int getCachedSamplingIntervalForOptimizeStats() {

        synchronized (mLock) {

            throwIfClosedLocked();

            return mBundleLocked.getInt(KEY_SAMPLING_INTERVAL_FOR_OPTIMIZE_STATS,

                    getCachedSamplingIntervalDefault());

        }

    }



    /** Returns the maximum serialized size an indexed document can be, in bytes. */

    public int getCachedLimitConfigMaxDocumentSizeBytes() {

        synchronized (mLock) {
@@ -343,6 +407,10 @@ public final class AppSearchConfig implements AutoCloseable { 
            case KEY_SAMPLING_INTERVAL_DEFAULT:

            case KEY_SAMPLING_INTERVAL_FOR_BATCH_CALL_STATS:

            case KEY_SAMPLING_INTERVAL_FOR_PUT_DOCUMENT_STATS:

            case KEY_SAMPLING_INTERVAL_FOR_INITIALIZE_STATS:

            case KEY_SAMPLING_INTERVAL_FOR_SEARCH_STATS:

            case KEY_SAMPLING_INTERVAL_FOR_GLOBAL_SEARCH_STATS:

            case KEY_SAMPLING_INTERVAL_FOR_OPTIMIZE_STATS:

                synchronized (mLock) {

                    mBundleLocked.putInt(key, properties.getInt(key, DEFAULT_SAMPLING_INTERVAL));

                }

apex/appsearch/service/java/com/android/server/appsearch/AppSearchManagerService.java

+25 −2
Original line number Diff line number Diff line
@@ -60,6 +60,7 @@ import com.android.internal.annotations.GuardedBy; 
import com.android.server.LocalManagerRegistry;

import com.android.server.SystemService;

import com.android.server.appsearch.external.localstorage.stats.CallStats;

import com.android.server.appsearch.external.localstorage.stats.OptimizeStats;

import com.android.server.appsearch.external.localstorage.visibilitystore.VisibilityStore;

import com.android.server.appsearch.stats.StatsCollector;

import com.android.server.appsearch.util.PackageUtil;
@@ -1497,20 +1498,42 @@ public class AppSearchManagerService extends SystemService { 


    private void checkForOptimize(AppSearchUserInstance instance, int mutateBatchSize) {

        EXECUTOR.execute(() -> {

            long totalLatencyStartMillis = SystemClock.elapsedRealtime();

            OptimizeStats.Builder builder = new OptimizeStats.Builder();

            try {

                instance.getAppSearchImpl().checkForOptimize(mutateBatchSize);

                instance.getAppSearchImpl().checkForOptimize(mutateBatchSize, builder);

            } catch (AppSearchException e) {

                Log.w(TAG, "Error occurred when check for optimize", e);

            } finally {

                OptimizeStats oStats = builder

                        .setTotalLatencyMillis(

                                (int) (SystemClock.elapsedRealtime() - totalLatencyStartMillis))

                        .build();

                if (oStats.getOriginalDocumentCount() > 0) {

                    // see if optimize has been run by checking originalDocumentCount

                    instance.getLogger().logStats(oStats);

                }

            }

        });

    }



    private void checkForOptimize(AppSearchUserInstance instance) {

        EXECUTOR.execute(() -> {

            long totalLatencyStartMillis = SystemClock.elapsedRealtime();

            OptimizeStats.Builder builder = new OptimizeStats.Builder();

            try {

                instance.getAppSearchImpl().checkForOptimize();

                instance.getAppSearchImpl().checkForOptimize(builder);

            } catch (AppSearchException e) {

                Log.w(TAG, "Error occurred when check for optimize", e);

            } finally {

                OptimizeStats oStats = builder

                        .setTotalLatencyMillis(

                                (int) (SystemClock.elapsedRealtime() - totalLatencyStartMillis))

                        .build();

                if (oStats.getOriginalDocumentCount() > 0) {

                    // see if optimize has been run by checking originalDocumentCount

                    instance.getLogger().logStats(oStats);

                }

            }

        });

    }

apex/appsearch/service/java/com/android/server/appsearch/external/localstorage/AppSearchImpl.java

+17 −7
Original line number Diff line number Diff line
@@ -55,6 +55,7 @@ import com.android.server.appsearch.external.localstorage.converter.SearchSpecTo 
import com.android.server.appsearch.external.localstorage.converter.SetSchemaResponseToProtoConverter;

import com.android.server.appsearch.external.localstorage.converter.TypePropertyPathToProtoConverter;

import com.android.server.appsearch.external.localstorage.stats.InitializeStats;

import com.android.server.appsearch.external.localstorage.stats.OptimizeStats;

import com.android.server.appsearch.external.localstorage.stats.PutDocumentStats;

import com.android.server.appsearch.external.localstorage.stats.RemoveStats;

import com.android.server.appsearch.external.localstorage.stats.SearchStats;
@@ -269,6 +270,9 @@ public final class AppSearchImpl implements Closeable { 


                // Log the time it took to read the data that goes into the cache maps

                if (initStatsBuilder != null) {

                    // In case there is some error for getAllNamespaces, we can still

                    // set the latency for preparation.

                    // If there is no error, the value will be overridden by the actual one later.

                    initStatsBuilder

                            .setStatusCode(

                                    statusProtoToResultCode(
@@ -1292,8 +1296,7 @@ public final class AppSearchImpl implements Closeable { 
                    deleteResultProto.getStatus(), StatusProto.Code.OK, StatusProto.Code.NOT_FOUND);



            // Update derived maps

            int numDocumentsDeleted =

                    deleteResultProto.getDeleteStats().getNumDocumentsDeleted();

            int numDocumentsDeleted = deleteResultProto.getDeleteStats().getNumDocumentsDeleted();

            updateDocumentCountAfterRemovalLocked(packageName, numDocumentsDeleted);

        } finally {

            mReadWriteLock.writeLock().unlock();
@@ -2096,12 +2099,13 @@ public final class AppSearchImpl implements Closeable { 
     *     #CHECK_OPTIMIZE_INTERVAL}, {@link IcingSearchEngine#getOptimizeInfo()} will be triggered

     *     and the counter will be reset.

     */

    public void checkForOptimize(int mutationSize) throws AppSearchException {

    public void checkForOptimize(int mutationSize, @Nullable OptimizeStats.Builder builder)

            throws AppSearchException {

        mReadWriteLock.writeLock().lock();

        try {

            mOptimizeIntervalCountLocked += mutationSize;

            if (mOptimizeIntervalCountLocked >= CHECK_OPTIMIZE_INTERVAL) {

                checkForOptimize();

                checkForOptimize(builder);

            }

        } finally {

            mReadWriteLock.writeLock().unlock();
@@ -2117,14 +2121,15 @@ public final class AppSearchImpl implements Closeable { 
     * <p>{@link IcingSearchEngine#optimize()} should be called only if {@link

     * OptimizeStrategy#shouldOptimize(GetOptimizeInfoResultProto)} return true.

     */

    public void checkForOptimize() throws AppSearchException {

    public void checkForOptimize(@Nullable OptimizeStats.Builder builder)

            throws AppSearchException {

        mReadWriteLock.writeLock().lock();

        try {

            GetOptimizeInfoResultProto optimizeInfo = getOptimizeInfoResultLocked();

            checkSuccess(optimizeInfo.getStatus());

            mOptimizeIntervalCountLocked = 0;

            if (mOptimizeStrategy.shouldOptimize(optimizeInfo)) {

                optimize();

                optimize(builder);

            }

        } finally {

            mReadWriteLock.writeLock().unlock();
@@ -2135,13 +2140,18 @@ public final class AppSearchImpl implements Closeable { 
    }



    /** Triggers {@link IcingSearchEngine#optimize()} directly. */

    public void optimize() throws AppSearchException {

    public void optimize(@Nullable OptimizeStats.Builder builder) throws AppSearchException {

        mReadWriteLock.writeLock().lock();

        try {

            mLogUtil.piiTrace("optimize, request");

            OptimizeResultProto optimizeResultProto = mIcingSearchEngineLocked.optimize();

            mLogUtil.piiTrace(

                    "optimize, response", optimizeResultProto.getStatus(), optimizeResultProto);

            if (builder != null) {

                builder.setStatusCode(statusProtoToResultCode(optimizeResultProto.getStatus()));

                AppSearchLoggerHelper.copyNativeStats(

                        optimizeResultProto.getOptimizeStats(), builder);

            }

            checkSuccess(optimizeResultProto.getStatus());

        } finally {

            mReadWriteLock.writeLock().unlock();

apex/appsearch/service/java/com/android/server/appsearch/external/localstorage/AppSearchLogger.java

+9 −6
Original line number Diff line number Diff line
@@ -17,10 +17,10 @@ 
package com.android.server.appsearch.external.localstorage;



import android.annotation.NonNull;

import android.app.appsearch.exceptions.AppSearchException;



import com.android.server.appsearch.external.localstorage.stats.CallStats;

import com.android.server.appsearch.external.localstorage.stats.InitializeStats;

import com.android.server.appsearch.external.localstorage.stats.OptimizeStats;

import com.android.server.appsearch.external.localstorage.stats.PutDocumentStats;

import com.android.server.appsearch.external.localstorage.stats.RemoveStats;

import com.android.server.appsearch.external.localstorage.stats.SearchStats;
@@ -37,19 +37,22 @@ import com.android.server.appsearch.external.localstorage.stats.SearchStats; 
 */

public interface AppSearchLogger {

    /** Logs {@link CallStats} */

    void logStats(@NonNull CallStats stats) throws AppSearchException;

    void logStats(@NonNull CallStats stats);



    /** Logs {@link PutDocumentStats} */

    void logStats(@NonNull PutDocumentStats stats) throws AppSearchException;

    void logStats(@NonNull PutDocumentStats stats);



    /** Logs {@link InitializeStats} */

    void logStats(@NonNull InitializeStats stats) throws AppSearchException;

    void logStats(@NonNull InitializeStats stats);



    /** Logs {@link SearchStats} */

    void logStats(@NonNull SearchStats stats) throws AppSearchException;

    void logStats(@NonNull SearchStats stats);



    /** Logs {@link RemoveStats} */

    void logStats(@NonNull RemoveStats stats) throws AppSearchException;

    void logStats(@NonNull RemoveStats stats);



    /** Logs {@link OptimizeStats} */

    void logStats(@NonNull OptimizeStats stats);



    // TODO(b/173532925) Add remaining logStats once we add all the stats.

}

apex/appsearch/service/java/com/android/server/appsearch/external/localstorage/AppSearchLoggerHelper.java

+30 −4
Original line number Diff line number Diff line
@@ -19,12 +19,14 @@ package com.android.server.appsearch.external.localstorage; 
import android.annotation.NonNull;



import com.android.server.appsearch.external.localstorage.stats.InitializeStats;

import com.android.server.appsearch.external.localstorage.stats.OptimizeStats;

import com.android.server.appsearch.external.localstorage.stats.PutDocumentStats;

import com.android.server.appsearch.external.localstorage.stats.RemoveStats;

import com.android.server.appsearch.external.localstorage.stats.SearchStats;



import com.google.android.icing.proto.DeleteStatsProto;

import com.google.android.icing.proto.InitializeStatsProto;

import com.google.android.icing.proto.OptimizeStatsProto;

import com.google.android.icing.proto.PutDocumentStatsProto;

import com.google.android.icing.proto.QueryStatsProto;
@@ -92,8 +94,8 @@ public final class AppSearchLoggerHelper { 
                .setSchemaTypeCount(fromNativeStats.getNumSchemaTypes());

    }



    /*

     * Copy native Query stats to buiilder.

    /**

     * Copies native Query stats to builder.

     *

     * @param fromNativeStats Stats copied from.

     * @param toStatsBuilder Stats copied to.
@@ -122,8 +124,8 @@ public final class AppSearchLoggerHelper { 
                        fromNativeStats.getDocumentRetrievalLatencyMs());

    }



    /*

     * Copy native Query stats to buiilder.

    /**

     * Copies native Delete stats to builder.

     *

     * @param fromNativeStats Stats copied from.

     * @param toStatsBuilder Stats copied to.
@@ -138,4 +140,28 @@ public final class AppSearchLoggerHelper { 
                .setDeleteType(fromNativeStats.getDeleteType().getNumber())

                .setDeletedDocumentCount(fromNativeStats.getNumDocumentsDeleted());

    }



    /**

     * Copies native {@link OptimizeStatsProto} to builder.

     *

     * @param fromNativeStats Stats copied from.

     * @param toStatsBuilder Stats copied to.

     */

    static void copyNativeStats(

            @NonNull OptimizeStatsProto fromNativeStats,

            @NonNull OptimizeStats.Builder toStatsBuilder) {

        Objects.requireNonNull(fromNativeStats);

        Objects.requireNonNull(toStatsBuilder);

        toStatsBuilder

                .setNativeLatencyMillis(fromNativeStats.getLatencyMs())

                .setDocumentStoreOptimizeLatencyMillis(

                        fromNativeStats.getDocumentStoreOptimizeLatencyMs())

                .setIndexRestorationLatencyMillis(fromNativeStats.getIndexRestorationLatencyMs())

                .setOriginalDocumentCount(fromNativeStats.getNumOriginalDocuments())

                .setDeletedDocumentCount(fromNativeStats.getNumDeletedDocuments())

                .setExpiredDocumentCount(fromNativeStats.getNumExpiredDocuments())

                .setStorageSizeBeforeBytes(fromNativeStats.getStorageSizeBefore())

                .setStorageSizeAfterBytes(fromNativeStats.getStorageSizeAfter())

                .setTimeSinceLastOptimizeMillis(fromNativeStats.getTimeSinceLastOptimizeMs());

    }

}