Merge "Implement schema migration in framework." into sc-dev am: 14fe7954ea (d9fa2171) · Commits · e / os / android_frameworks_base

apex/appsearch/framework/java/android/app/appsearch/AppSearchMigrationHelper.java

0 → 100644

+256 −0

Original line number	Diff line number	Diff line
		/*
		* Copyright (C) 2021 The Android Open Source Project
		*
		* Licensed under the Apache License, Version 2.0 (the "License");
		* you may not use this file except in compliance with the License.
		* You may obtain a copy of the License at
		*
		* http://www.apache.org/licenses/LICENSE-2.0
		*
		* Unless required by applicable law or agreed to in writing, software
		* distributed under the License is distributed on an "AS IS" BASIS,
		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
		* See the License for the specific language governing permissions and
		* limitations under the License.
		*/

		package android.app.appsearch;

		import static android.os.ParcelFileDescriptor.MODE_READ_ONLY;
		import static android.os.ParcelFileDescriptor.MODE_WRITE_ONLY;

		import android.annotation.NonNull;
		import android.annotation.UserIdInt;
		import android.annotation.WorkerThread;
		import android.app.appsearch.exceptions.AppSearchException;
		import android.os.Bundle;
		import android.os.Parcel;
		import android.os.ParcelFileDescriptor;
		import android.os.RemoteException;

		import com.android.internal.infra.AndroidFuture;

		import java.io.Closeable;
		import java.io.DataInputStream;
		import java.io.DataOutputStream;
		import java.io.EOFException;
		import java.io.File;
		import java.io.FileInputStream;
		import java.io.FileOutputStream;
		import java.io.IOException;
		import java.util.List;
		import java.util.Map;
		import java.util.Objects;
		import java.util.concurrent.ExecutionException;

		/**
		* The helper class for {@link AppSearchSchema} migration.
		*
		* <p>It will query and migrate {@link GenericDocument} in given type to a new version.
		* @hide
		*/
		public class AppSearchMigrationHelper implements Closeable {
		private final IAppSearchManager mService;
		private final String mPackageName;
		private final String mDatabaseName;
		private final int mUserId;
		private final File mMigratedFile;
		private final Map<String, Integer> mCurrentVersionMap;
		private final Map<String, Integer> mFinalVersionMap;
		private boolean mAreDocumentsMigrated = false;

		AppSearchMigrationHelper(@NonNull IAppSearchManager service,
		@UserIdInt int userId,
		@NonNull Map<String, Integer> currentVersionMap,
		@NonNull Map<String, Integer> finalVersionMap,
		@NonNull String packageName,
		@NonNull String databaseName) throws IOException {
		mService = Objects.requireNonNull(service);
		mCurrentVersionMap = Objects.requireNonNull(currentVersionMap);
		mFinalVersionMap = Objects.requireNonNull(finalVersionMap);
		mPackageName = Objects.requireNonNull(packageName);
		mDatabaseName = Objects.requireNonNull(databaseName);
		mUserId = userId;
		mMigratedFile = File.createTempFile(/prefix=/"appsearch", /suffix=/null);
		}

		/**
		* Queries all documents that need to be migrated to a different version and transform
		* documents to that version by passing them to the provided {@link Migrator}.
		*
		* <p>The method will be executed on the executor provided to
		* {@link AppSearchSession#setSchema}.
		*
		* @param schemaType The schema type that needs to be updated and whose {@link GenericDocument}
		* need to be migrated.
		* @param migrator The {@link Migrator} that will upgrade or downgrade a {@link
		* GenericDocument} to new version.
		*/
		@WorkerThread
		public void queryAndTransform(@NonNull String schemaType, @NonNull Migrator migrator)
		throws IOException, AppSearchException, InterruptedException, ExecutionException {
		File queryFile = File.createTempFile(/prefix=/"appsearch", /suffix=/null);
		try (ParcelFileDescriptor fileDescriptor =
		ParcelFileDescriptor.open(queryFile, MODE_WRITE_ONLY)) {
		AndroidFuture<AppSearchResult<Void>> androidFuture = new AndroidFuture<>();
		mService.writeQueryResultsToFile(mPackageName, mDatabaseName,
		fileDescriptor,
		/queryExpression=/ "",
		new SearchSpec.Builder()
		.addFilterSchemas(schemaType)
		.setTermMatch(SearchSpec.TERM_MATCH_EXACT_ONLY)
		.build().getBundle(),
		mUserId,
		new IAppSearchResultCallback.Stub() {
		@Override
		public void onResult(AppSearchResult result) throws RemoteException {
		androidFuture.complete(result);
		}
		});
		AppSearchResult<Void> result = androidFuture.get();
		if (!result.isSuccess()) {
		throw new AppSearchException(result.getResultCode(), result.getErrorMessage());
		}
		readAndTransform(queryFile, migrator);
		} catch (RemoteException e) {
		throw e.rethrowFromSystemServer();
		} finally {
		queryFile.delete();
		}
		}

		/**
		* Puts all {@link GenericDocument} migrated from the previous call to
		* {@link #queryAndTransform} into AppSearch.
		*
		* <p> This method should be only called once.
		*
		* @param responseBuilder a SetSchemaResponse builder whose result will be returned by this
		* function with any
		* {@link android.app.appsearch.SetSchemaResponse.MigrationFailure}
		* added in.
		* @return the {@link SetSchemaResponse} for {@link AppSearchSession#setSchema} call.
		*/
		@NonNull
		AppSearchResult<SetSchemaResponse> putMigratedDocuments(
		@NonNull SetSchemaResponse.Builder responseBuilder) {
		if (!mAreDocumentsMigrated) {
		return AppSearchResult.newSuccessfulResult(responseBuilder.build());
		}
		try (ParcelFileDescriptor fileDescriptor =
		ParcelFileDescriptor.open(mMigratedFile, MODE_READ_ONLY)) {
		AndroidFuture<AppSearchResult<List<Bundle>>> androidFuture = new AndroidFuture<>();
		mService.putDocumentsFromFile(mPackageName, mDatabaseName, fileDescriptor, mUserId,
		new IAppSearchResultCallback.Stub() {
		@Override
		public void onResult(AppSearchResult result) throws RemoteException {
		androidFuture.complete(result);
		}
		});
		AppSearchResult<List<Bundle>> result = androidFuture.get();
		if (!result.isSuccess()) {
		return AppSearchResult.newFailedResult(result);
		}
		List<Bundle> migratedFailureBundles = result.getResultValue();
		for (int i = 0; i < migratedFailureBundles.size(); i++) {
		responseBuilder.addMigrationFailure(
		new SetSchemaResponse.MigrationFailure(migratedFailureBundles.get(i)));
		}
		} catch (RemoteException e) {
		throw e.rethrowFromSystemServer();
		} catch (Throwable t) {
		return AppSearchResult.throwableToFailedResult(t);
		} finally {
		mMigratedFile.delete();
		}
		return AppSearchResult.newSuccessfulResult(responseBuilder.build());
		}

		/**
		* Reads all saved {@link GenericDocument}s from the given {@link File}.
		*
		* <p>Transforms those {@link GenericDocument}s to the final version.
		*
		* <p>Save migrated {@link GenericDocument}s to the {@link #mMigratedFile}.
		*/
		private void readAndTransform(@NonNull File file, @NonNull Migrator migrator)
		throws IOException {
		try (DataInputStream inputStream = new DataInputStream(new FileInputStream(file));
		DataOutputStream outputStream = new DataOutputStream(new FileOutputStream(
		mMigratedFile, /append=/ true))) {
		GenericDocument document;
		while (true) {
		try {
		document = readDocumentFromInputStream(inputStream);
		} catch (EOFException e) {
		break;
		// Nothing wrong. We just finished reading.
		}

		int currentVersion = mCurrentVersionMap.get(document.getSchemaType());
		int finalVersion = mFinalVersionMap.get(document.getSchemaType());

		GenericDocument newDocument;
		if (currentVersion < finalVersion) {
		newDocument = migrator.onUpgrade(currentVersion, finalVersion, document);
		} else {
		// currentVersion == finalVersion case won't trigger migration and get here.
		newDocument = migrator.onDowngrade(currentVersion, finalVersion, document);
		}
		writeBundleToOutputStream(outputStream, newDocument.getBundle());
		}
		mAreDocumentsMigrated = true;
		}
		}

		/**
		* Reads the {@link Bundle} of a {@link GenericDocument} from given {@link DataInputStream}.
		*
		* @param inputStream The inputStream to read from
		*
		* @throws IOException on read failure.
		* @throws EOFException if {@link java.io.InputStream} reaches the end.
		*/
		@NonNull
		public static GenericDocument readDocumentFromInputStream(
		@NonNull DataInputStream inputStream) throws IOException {
		int length = inputStream.readInt();
		if (length == 0) {
		throw new EOFException();
		}
		byte[] serializedMessage = new byte[length];
		inputStream.read(serializedMessage);

		Parcel parcel = Parcel.obtain();
		try {
		parcel.unmarshall(serializedMessage, 0, serializedMessage.length);
		parcel.setDataPosition(0);
		Bundle bundle = parcel.readBundle();
		return new GenericDocument(bundle);
		} finally {
		parcel.recycle();
		}
		}

		/**
		* Serializes a {@link Bundle} and writes into the given {@link DataOutputStream}.
		*/
		public static void writeBundleToOutputStream(
		@NonNull DataOutputStream outputStream, @NonNull Bundle bundle)
		throws IOException {
		Parcel parcel = Parcel.obtain();
		try {
		parcel.writeBundle(bundle);
		byte[] serializedMessage = parcel.marshall();
		outputStream.writeInt(serializedMessage.length);
		outputStream.write(serializedMessage);
		} finally {
		parcel.recycle();
		}
		}

		@Override
		public void close() throws IOException {
		mMigratedFile.delete();
		}
		}

apex/appsearch/framework/java/android/app/appsearch/AppSearchSession.java

+208 −8

Original line number	Diff line number	Diff line
		@@ -19,6 +19,8 @@ package android.app.appsearch;
		import android.annotation.CallbackExecutor;
		import android.annotation.NonNull;
		import android.annotation.UserIdInt;
		import android.app.appsearch.exceptions.AppSearchException;
		import android.app.appsearch.util.SchemaMigrationUtil;
		import android.os.Bundle;
		import android.os.ParcelableException;
		import android.os.RemoteException;
		@@ -26,14 +28,17 @@ import android.util.ArrayMap;
		import android.util.ArraySet;
		import android.util.Log;

		import com.android.internal.infra.AndroidFuture;
		import com.android.internal.util.Preconditions;

		import java.io.Closeable;
		import java.io.IOException;
		import java.util.ArrayList;
		import java.util.List;
		import java.util.Map;
		import java.util.Objects;
		import java.util.Set;
		import java.util.concurrent.ExecutionException;
		import java.util.concurrent.Executor;
		import java.util.function.Consumer;

		@@ -55,7 +60,6 @@ public final class AppSearchSession implements Closeable {
		private boolean mIsMutated = false;
		private boolean mIsClosed = false;


		/**
		* Creates a search session for the client, defined by the {@code userId} and
		* {@code packageName}.
		@@ -157,26 +161,65 @@ public final class AppSearchSession implements Closeable {
		}
		schemasPackageAccessibleBundles.put(entry.getKey(), packageIdentifierBundles);
		}

		// No need to trigger migration if user never set migrator
		if (request.getMigrators().isEmpty()) {
		setSchemaNoMigrations(
		request,
		schemaBundles,
		schemasPackageAccessibleBundles,
		callbackExecutor,
		callback);
		return;
		}

		try {
		// Migration process
		// 1. Generate the current and the final version map.
		// TODO(b/182855402) Release binder thread and move the heavy work into worker thread.
		AndroidFuture<AppSearchResult<GetSchemaResponse>> future = new AndroidFuture<>();
		getSchema(callbackExecutor, future::complete);
		AppSearchResult<GetSchemaResponse> getSchemaResult = future.get();
		if (!getSchemaResult.isSuccess()) {
		callback.accept(AppSearchResult.newFailedResult(getSchemaResult));
		return;
		}
		GetSchemaResponse getSchemaResponse = getSchemaResult.getResultValue();
		Set<AppSearchSchema> currentSchemas = getSchemaResponse.getSchemas();
		Map<String, Integer> currentVersionMap =
		SchemaMigrationUtil.buildVersionMap(currentSchemas,
		getSchemaResponse.getVersion());
		Map<String, Integer> finalVersionMap =
		SchemaMigrationUtil.buildVersionMap(request.getSchemas(), request.getVersion());

		// 2. SetSchema with forceOverride=false, to retrieve the list of incompatible/deleted
		// types.
		mService.setSchema(
		mPackageName,
		mDatabaseName,
		schemaBundles,
		new ArrayList<>(request.getSchemasNotDisplayedBySystem()),
		schemasPackageAccessibleBundles,
		request.isForceOverride(),
		/forceOverride=/ false,
		mUserId,
		request.getVersion(),
		new IAppSearchResultCallback.Stub() {
		public void onResult(AppSearchResult result) {
		callbackExecutor.execute(() -> {
		if (result.isSuccess()) {
		callback.accept(
		// TODO(b/177266929) implement Migration in platform.
		// TODO(b/183177268): once migration is implemented, run
		// it on workExecutor.
		AppSearchResult.newSuccessfulResult(
		new SetSchemaResponse.Builder().build()));
		try {
		Bundle bundle = (Bundle) result.getResultValue();
		SetSchemaResponse setSchemaResponse =
		new SetSchemaResponse(bundle);
		setSchemaMigration(
		request, setSchemaResponse, schemaBundles,
		schemasPackageAccessibleBundles, currentVersionMap,
		finalVersionMap, callback);
		} catch (Throwable t) {
		callback.accept(AppSearchResult.throwableToFailedResult(t));
		}
		} else {
		callback.accept(result);
		}
		@@ -186,6 +229,8 @@ public final class AppSearchSession implements Closeable {
		mIsMutated = true;
		} catch (RemoteException e) {
		throw e.rethrowFromSystemServer();
		} catch (Throwable t) {
		callback.accept(AppSearchResult.throwableToFailedResult(t));
		}
		}

		@@ -627,4 +672,159 @@ public final class AppSearchSession implements Closeable {
		}
		}
		}

		/**
		* Set schema to Icing for no-migration scenario.
		*
		* <p>We only need one time {@link #setSchema} call for no-migration scenario by using the
		* forceoverride in the request.
		*/
		private void setSchemaNoMigrations(@NonNull SetSchemaRequest request,
		@NonNull List<Bundle> schemaBundles,
		@NonNull Map<String, List<Bundle>> schemasPackageAccessibleBundles,
		@NonNull @CallbackExecutor Executor executor,
		@NonNull Consumer<AppSearchResult<SetSchemaResponse>> callback) {
		try {
		mService.setSchema(
		mPackageName,
		mDatabaseName,
		schemaBundles,
		new ArrayList<>(request.getSchemasNotDisplayedBySystem()),
		schemasPackageAccessibleBundles,
		request.isForceOverride(),
		mUserId,
		request.getVersion(),
		new IAppSearchResultCallback.Stub() {
		public void onResult(AppSearchResult result) {
		executor.execute(() -> {
		if (result.isSuccess()) {
		try {
		SetSchemaResponse setSchemaResponse =
		new SetSchemaResponse(
		(Bundle) result.getResultValue());
		if (!request.isForceOverride()) {
		// Throw exception if there is any deleted types or
		// incompatible types. That's the only case we swallowed
		// in the AppSearchImpl#setSchema().
		checkDeletedAndIncompatible(
		setSchemaResponse.getDeletedTypes(),
		setSchemaResponse.getIncompatibleTypes());
		}
		callback.accept(AppSearchResult
		.newSuccessfulResult(setSchemaResponse));
		} catch (Throwable t) {
		callback.accept(AppSearchResult.throwableToFailedResult(t));
		}
		} else {
		callback.accept(result);
		}
		});
		}
		});
		mIsMutated = true;
		} catch (RemoteException e) {
		throw e.rethrowFromSystemServer();
		}
		}

		/**
		* Set schema to Icing for migration scenario.
		*
		* <p>First time {@link #setSchema} call with forceOverride is false gives us all incompatible
		* changes. After trigger migrations, the second time call {@link #setSchema} will actually
		* apply the changes.
		*
		* @param setSchemaResponse the result of the first setSchema call with forceOverride=false.
		*/
		private void setSchemaMigration(@NonNull SetSchemaRequest request,
		@NonNull SetSchemaResponse setSchemaResponse,
		@NonNull List<Bundle> schemaBundles,
		@NonNull Map<String, List<Bundle>> schemasPackageAccessibleBundles,
		@NonNull Map<String, Integer> currentVersionMap, Map<String, Integer> finalVersionMap,
		@NonNull Consumer<AppSearchResult<SetSchemaResponse>> callback)
		throws AppSearchException, IOException, RemoteException, ExecutionException,
		InterruptedException {
		// 1. If forceOverride is false, check that all incompatible types will be migrated.
		// If some aren't we must throw an error, rather than proceeding and deleting those
		// types.
		if (!request.isForceOverride()) {
		Set<String> unmigratedTypes = SchemaMigrationUtil.getUnmigratedIncompatibleTypes(
		setSchemaResponse.getIncompatibleTypes(),
		request.getMigrators(),
		currentVersionMap,
		finalVersionMap);
		// check if there are any unmigrated types or deleted types. If there are, we will throw
		// an exception.
		// Since the force override is false, the schema will not have been set if there are any
		// incompatible or deleted types.
		checkDeletedAndIncompatible(setSchemaResponse.getDeletedTypes(),
		unmigratedTypes);
		}

		try (AppSearchMigrationHelper migrationHelper =
		new AppSearchMigrationHelper(mService, mUserId, currentVersionMap,
		finalVersionMap, mPackageName, mDatabaseName)) {
		Map<String, Migrator> migratorMap = request.getMigrators();

		// 2. Trigger migration for all migrators.
		// TODO(b/177266929) trigger migration for all types together rather than separately.
		Set<String> migratedTypes = new ArraySet<>();
		for (Map.Entry<String, Migrator> entry : migratorMap.entrySet()) {
		String schemaType = entry.getKey();
		Migrator migrator = entry.getValue();
		if (SchemaMigrationUtil.shouldTriggerMigration(
		schemaType, migrator, currentVersionMap, finalVersionMap)) {
		migrationHelper.queryAndTransform(schemaType, migrator);
		migratedTypes.add(schemaType);
		}
		}

		// 3. SetSchema a second time with forceOverride=true if the first attempted failed.
		if (!setSchemaResponse.getIncompatibleTypes().isEmpty()
		\|\| !setSchemaResponse.getDeletedTypes().isEmpty()) {
		AndroidFuture<AppSearchResult<SetSchemaResponse>> future = new AndroidFuture<>();
		// only trigger second setSchema() call if the first one is fail.
		mService.setSchema(
		mPackageName,
		mDatabaseName,
		schemaBundles,
		new ArrayList<>(request.getSchemasNotDisplayedBySystem()),
		schemasPackageAccessibleBundles,
		/forceOverride=/ true,
		mUserId,
		request.getVersion(),
		new IAppSearchResultCallback.Stub() {
		@Override
		public void onResult(AppSearchResult result) throws RemoteException {
		future.complete(result);
		}
		});
		AppSearchResult<SetSchemaResponse> secondSetSchemaResult = future.get();
		if (!secondSetSchemaResult.isSuccess()) {
		// we failed to set the schema in second time with force override = true, which
		// is an impossible case. Since we only swallow the incompatible error in the
		// first setSchema call, all other errors will be thrown at the first time.
		callback.accept(secondSetSchemaResult);
		return;
		}
		}

		SetSchemaResponse.Builder responseBuilder = setSchemaResponse.toBuilder()
		.addMigratedTypes(migratedTypes);
		callback.accept(migrationHelper.putMigratedDocuments(responseBuilder));
		}
		}

		/** Checks the setSchema() call won't delete any types or has incompatible types. */
		//TODO(b/177266929) move this method to util
		private void checkDeletedAndIncompatible(Set<String> deletedTypes,
		Set<String> incompatibleTypes)
		throws AppSearchException {
		if (!deletedTypes.isEmpty() \|\| !incompatibleTypes.isEmpty()) {
		String newMessage = "Schema is incompatible."
		+ "\n Deleted types: " + deletedTypes
		+ "\n Incompatible types: " + incompatibleTypes;
		throw new AppSearchException(AppSearchResult.RESULT_INVALID_SCHEMA, newMessage);
		}
		}
		}

apex/appsearch/framework/java/android/app/appsearch/IAppSearchManager.aidl

+44 −1

File changed.

Preview size limit exceeded, changes collapsed.

apex/appsearch/framework/java/external/android/app/appsearch/util/SchemaMigrationUtil.java

+67 −52

File changed.

Preview size limit exceeded, changes collapsed.

apex/appsearch/service/java/com/android/server/appsearch/AppSearchManagerService.java

+103 −3

File changed.

Preview size limit exceeded, changes collapsed.