Merge "Allow PM to scan directories in parallel." into main (13aa74f9) · Commits · e / os / android_frameworks_base

services/core/java/com/android/server/pm/InitAppsHelper.java

+86 −21

Original line number	Diff line number	Diff line
		@@ -175,7 +175,11 @@ final class InitAppsHelper {
		final List<ApexManager.ScanResult> apexScanResults = scanApexPackagesTraced(packageParser);
		mApexManager.notifyScanResult(apexScanResults);

		scanSystemDirs(packageParser, mExecutorService);
		// If this flag is enabled, system directories will be scanned in parallel. This doesn't
		// affect the order that packages are installed in.
		scanSystemDirs(packageParser, mExecutorService,
		android.content.pm.Flags.parallelPackageParsingAcrossSystemDirs());

		// Parse overlay configuration files to set default enable state, mutability, and
		// priority of system overlays.
		final ArrayMap<String, File> apkInApexPreInstalledPaths = new ArrayMap<>();
		@@ -310,12 +314,59 @@ final class InitAppsHelper {
		}
		}

		static class ScanParams {
		public final @Nullable File scanDir;
		public final int parseFlags;
		public final int scanFlags;
		public final @Nullable ApexManager.ActiveApexInfo apexInfo;

		private ScanParams(File scanDir, int parseFlags, int scanFlags,
		ApexManager.ActiveApexInfo apexInfo) {
		this.scanDir = scanDir;
		this.parseFlags = parseFlags;
		this.scanFlags = scanFlags;
		this.apexInfo = apexInfo;
		}

		public static ScanParams forApexDirScan(int parseFlags, int scanFlags) {
		return new ScanParams(null, parseFlags, scanFlags, null);
		}

		public static ScanParams forApkPartitionScan(File scanDir, int parseFlags, int scanFlags) {
		return new ScanParams(scanDir, parseFlags, scanFlags, null);
		}

		public static ScanParams forApkInApexScan(File scanDir, int parseFlags, int scanFlags,
		ApexManager.ActiveApexInfo apexInfo) {
		// When scanning apk in apexes, we want to check the maxSdkVersion.
		return new ScanParams(scanDir, parseFlags \| PARSE_APK_IN_APEX, scanFlags, apexInfo);
		}
		}

		// Helper function for either immediately scanning scanDir using the input scan parameters, or
		// just collecting them in scanParamsList to be processed in parallel later.
		@GuardedBy({"mPm.mInstallLock", "mPm.mLock"})
		private void processOrCollectScanParams(List<ScanParams> scanParamsList, File scanDir,
		int parseFlags, int scanFlags, PackageParser2 packageParser,
		ExecutorService executorService, ApexManager.ActiveApexInfo apexInfo) {
		if (scanParamsList == null) {
		scanDirTracedLI(scanDir, parseFlags, scanFlags, packageParser, executorService,
		apexInfo);
		} else {
		scanParamsList.add((scanFlags & SCAN_AS_APK_IN_APEX) != 0
		? ScanParams.forApkInApexScan(scanDir, parseFlags, scanFlags, apexInfo)
		: ScanParams.forApkPartitionScan(scanDir, parseFlags, scanFlags));
		}
		}

		/**
		* First part of init dir scanning
		*/
		@GuardedBy({"mPm.mInstallLock", "mPm.mLock"})
		private void scanSystemDirs(PackageParser2 packageParser, ExecutorService executorService) {
		private void scanSystemDirs(PackageParser2 packageParser, ExecutorService executorService,
		boolean parallelDirScan) {
		File frameworkDir = new File(Environment.getRootDirectory(), "framework");
		List<ScanParams> scanParamsList = parallelDirScan ? new ArrayList<>() : null;

		// Collect vendor/product/system_ext overlay packages. (Do this before scanning
		// any apps.)
		@@ -326,30 +377,36 @@ final class InitAppsHelper {
		if (partition.getOverlayFolder() == null) {
		continue;
		}
		scanDirTracedLI(partition.getOverlayFolder(),
		mSystemParseFlags, mSystemScanFlags \| partition.scanFlag,
		packageParser, executorService, partition.apexInfo);
		}

		scanDirTracedLI(frameworkDir,
		mSystemParseFlags, mSystemScanFlags \| SCAN_NO_DEX \| SCAN_AS_PRIVILEGED,
		packageParser, executorService, null);
		if (!mPm.mPackages.containsKey("android")) {
		throw new IllegalStateException(
		"Failed to load frameworks package; check log for warnings");
		processOrCollectScanParams(scanParamsList, partition.getOverlayFolder(),
		mSystemParseFlags, mSystemScanFlags \| partition.scanFlag, packageParser,
		executorService, partition.apexInfo);
		}
		processOrCollectScanParams(scanParamsList, frameworkDir, mSystemParseFlags,
		mSystemScanFlags \| SCAN_NO_DEX \| SCAN_AS_PRIVILEGED, packageParser,
		executorService, null);

		for (int i = 0, size = mDirsToScanAsSystem.size(); i < size; i++) {
		final ScanPartition partition = mDirsToScanAsSystem.get(i);
		if (partition.getPrivAppFolder() != null) {
		scanDirTracedLI(partition.getPrivAppFolder(),
		processOrCollectScanParams(scanParamsList, partition.getPrivAppFolder(),
		mSystemParseFlags,
		mSystemScanFlags \| SCAN_AS_PRIVILEGED \| partition.scanFlag,
		packageParser, executorService, partition.apexInfo);
		}
		scanDirTracedLI(partition.getAppFolder(),
		mSystemParseFlags, mSystemScanFlags \| partition.scanFlag,
		packageParser, executorService, partition.apexInfo);
		processOrCollectScanParams(scanParamsList, partition.getAppFolder(), mSystemParseFlags,
		mSystemScanFlags \| partition.scanFlag, packageParser, executorService,
		partition.apexInfo);
		}

		// Scan all directories with the parameters contained in scanParamsList. If parallelDirScan
		// is not enabled, we will have already scanned and processed everything by this point.
		if (parallelDirScan) {
		parallelScanDirTracedLI(scanParamsList, packageParser, executorService);
		}

		if (!mPm.mPackages.containsKey("android")) {
		throw new IllegalStateException(
		"Failed to load frameworks package; check log for warnings");
		}
		}

		@@ -370,10 +427,6 @@ final class InitAppsHelper {
		@Nullable ApexManager.ActiveApexInfo apexInfo) {
		Trace.traceBegin(TRACE_TAG_PACKAGE_MANAGER, "scanDir [" + scanDir.getAbsolutePath() + "]");
		try {
		if ((scanFlags & SCAN_AS_APK_IN_APEX) != 0) {
		// when scanning apk in apexes, we want to check the maxSdkVersion
		parseFlags \|= PARSE_APK_IN_APEX;
		}
		mInstallPackageHelper.installPackagesFromDir(scanDir, parseFlags,
		scanFlags, packageParser, executorService, apexInfo);
		} finally {
		@@ -381,6 +434,18 @@ final class InitAppsHelper {
		}
		}

		@GuardedBy({"mPm.mInstallLock", "mPm.mLock"})
		private void parallelScanDirTracedLI(List<ScanParams> scanParamsList,
		PackageParser2 packageParser, ExecutorService executorService) {
		Trace.traceBegin(TRACE_TAG_PACKAGE_MANAGER, "parallelScanDir");
		try {
		mInstallPackageHelper.parallelInstallPackagesFromDirs(scanParamsList,
		packageParser, executorService);
		} finally {
		Trace.traceEnd(TRACE_TAG_PACKAGE_MANAGER);
		}
		}

		public boolean isExpectingBetter(String packageName) {
		return mExpectingBetter.containsKey(packageName);
		}

services/core/java/com/android/server/pm/InstallPackageHelper.java

+130 −44

Original line number	Diff line number	Diff line
		@@ -47,6 +47,7 @@ import static android.os.storage.StorageManager.FLAG_STORAGE_CE;
		import static android.os.storage.StorageManager.FLAG_STORAGE_DE;
		import static android.os.storage.StorageManager.FLAG_STORAGE_EXTERNAL;

		import static com.android.server.pm.InitAppsHelper.ScanParams;
		import static com.android.server.pm.InstructionSets.getAppDexInstructionSets;
		import static com.android.server.pm.PackageManagerException.INTERNAL_ERROR_ARCHIVE_NO_INSTALLER_TITLE;
		import static com.android.server.pm.PackageManagerService.APP_METADATA_FILE_NAME;
		@@ -94,6 +95,7 @@ import static com.android.server.pm.PackageManagerServiceUtils.extractAppMetadat
		import static com.android.server.pm.PackageManagerServiceUtils.isInstalledByAdb;
		import static com.android.server.pm.PackageManagerServiceUtils.logCriticalInfo;
		import static com.android.server.pm.PackageManagerServiceUtils.makeDirRecursive;
		import static com.android.server.pm.ParallelPackageParser.OrderedResult;
		import static com.android.server.pm.SharedUidMigration.BEST_EFFORT;

		import android.annotation.NonNull;
		@@ -205,6 +207,7 @@ import java.util.List;
		import java.util.Map;
		import java.util.Set;
		import java.util.concurrent.CompletableFuture;
		import java.util.concurrent.ExecutionException;
		import java.util.concurrent.ExecutorService;


		@@ -3753,12 +3756,13 @@ final class InstallPackageHelper {

		ParallelPackageParser parallelPackageParser =
		new ParallelPackageParser(packageParser, executorService);
		ScanParams scanParams = ScanParams.forApexDirScan(parseFlags, scanFlags);

		// Submit files for parsing in parallel
		ArrayMap<File, ApexInfo> parsingApexInfo = new ArrayMap<>();
		for (ApexInfo ai : allPackages) {
		File apexFile = new File(ai.modulePath);
		parallelPackageParser.submit(apexFile, parseFlags);
		parallelPackageParser.submit(apexFile, scanParams);
		parsingApexInfo.put(apexFile, ai);
		}

		@@ -3817,18 +3821,61 @@ final class InstallPackageHelper {
		public void installPackagesFromDir(File scanDir, int parseFlags,
		int scanFlags, PackageParser2 packageParser, ExecutorService executorService,
		@Nullable ApexManager.ActiveApexInfo apexInfo) {
		final File[] files = scanDir.listFiles();
		ParallelPackageParser parallelPackageParser =
		new ParallelPackageParser(packageParser, executorService);
		ScanParams scanParams = (scanFlags & SCAN_AS_APK_IN_APEX) != 0
		? ScanParams.forApkInApexScan(scanDir, parseFlags, scanFlags, apexInfo)
		: ScanParams.forApkPartitionScan(scanDir, parseFlags, scanFlags);
		int fileCount = scanDirectoryForFilesToParse(parallelPackageParser, scanParams);

		// Process results one by one
		for (; fileCount > 0; fileCount--) {
		processParseResult(parallelPackageParser.take());
		}
		}

		/**
		* Performs scans across multiple directories in parallel, then installs packages in the order
		* that scans were submitted.
		*/
		@GuardedBy({"mPm.mInstallLock", "mPm.mLock"})
		public void parallelInstallPackagesFromDirs(List<ScanParams> scanParamsList,
		PackageParser2 packageParser, ExecutorService executorService) {
		ParallelPackageParser parallelPackageParser =
		new ParallelPackageParser(packageParser, executorService);
		List<List<OrderedResult>> resultsList = new ArrayList<>();

		// Submit scan and parse tasks across all directories.
		for (ScanParams scanParams : scanParamsList) {
		resultsList.add(orderedScanDirectoryForFilesToParse(parallelPackageParser, scanParams));
		}

		// Process results in order.
		for (List<OrderedResult> partitionResults : resultsList) {
		for (OrderedResult result : partitionResults) {
		try {
		processParseResult(result.future.get());
		} catch (ExecutionException \| InterruptedException e) {
		throw new IllegalStateException("Unable to parse: " + result.scanFile);
		}
		}
		}
		}

		@GuardedBy({"mPm.mInstallLock", "mPm.mLock"})
		private int scanDirectoryForFilesToParse(ParallelPackageParser parallelPackageParser,
		ScanParams scanParams) {
		final File[] files = scanParams.scanDir.listFiles();
		if (ArrayUtils.isEmpty(files)) {
		Log.d(TAG, "No files in app dir " + scanDir);
		return;
		Log.d(TAG, "No files in app dir " + scanParams.scanDir);
		return 0;
		}

		if (DEBUG_PACKAGE_SCANNING) {
		Log.d(TAG, "Scanning app dir " + scanDir + " scanFlags=" + scanFlags
		+ " flags=0x" + Integer.toHexString(parseFlags));
		Log.d(TAG, "Scanning app dir " + scanParams.scanDir + " scanFlags = "
		+ scanParams.scanFlags + " flags=0x"
		+ Integer.toHexString(scanParams.parseFlags));
		}
		ParallelPackageParser parallelPackageParser =
		new ParallelPackageParser(packageParser, executorService);

		// Submit files for parsing in parallel
		int fileCount = 0;
		@@ -3839,31 +3886,71 @@ final class InstallPackageHelper {
		// Ignore entries which are not packages
		continue;
		}
		if ((scanFlags & SCAN_DROP_CACHE) != 0) {
		if ((scanParams.scanFlags & SCAN_DROP_CACHE) != 0) {
		final PackageCacher cacher = new PackageCacher(mPm.getCacheDir(),
		mPm.mPackageParserCallback);
		Log.w(TAG, "Dropping cache of " + file.getAbsolutePath());
		cacher.cleanCachedResult(file);
		}
		parallelPackageParser.submit(file, parseFlags);
		parallelPackageParser.submit(file, scanParams);
		fileCount++;
		}
		return fileCount;
		}

		// Process results one by one
		for (; fileCount > 0; fileCount--) {
		ParallelPackageParser.ParseResult parseResult = parallelPackageParser.take();
		Throwable throwable = parseResult.throwable;
		// This is similar to scanDirectoryForFilesToParse, but instead of collecting parse tasks in the
		// order they return (and returning the number of files scanned), this returns the list of parse
		// results in the order that scans were submitted.
		@GuardedBy({"mPm.mInstallLock", "mPm.mLock"})
		private List<OrderedResult> orderedScanDirectoryForFilesToParse(
		ParallelPackageParser parallelPackageParser, ScanParams scanParams) {
		List<OrderedResult> orderedResults = new ArrayList<>();
		final File[] files = scanParams.scanDir.listFiles();
		if (ArrayUtils.isEmpty(files)) {
		Log.d(TAG, "No files in app dir " + scanParams.scanDir);
		return orderedResults;
		}

		if (DEBUG_PACKAGE_SCANNING) {
		Log.d(TAG, "Scanning app dir " + scanParams.scanDir + " scanFlags = "
		+ scanParams.scanFlags + " flags=0x"
		+ Integer.toHexString(scanParams.parseFlags));
		}

		// Submit files for parsing in parallel
		for (File file : files) {
		final boolean isPackage = (isApkFile(file) \|\| file.isDirectory())
		&& !PackageInstallerService.isStageName(file.getName());
		if (!isPackage) {
		// Ignore entries which are not packages
		continue;
		}
		if ((scanParams.scanFlags & SCAN_DROP_CACHE) != 0) {
		final PackageCacher cacher = new PackageCacher(mPm.getCacheDir(),
		mPm.mPackageParserCallback);
		Log.w(TAG, "Dropping cache of " + file.getAbsolutePath());
		cacher.cleanCachedResult(file);
		}
		orderedResults.add(parallelPackageParser.orderedSubmit(file, scanParams));
		}
		return orderedResults;
		}

		@GuardedBy({"mPm.mInstallLock", "mPm.mLock"})
		private void processParseResult(ParallelPackageParser.ParseResult result) {
		Throwable throwable = result.throwable;
		int errorCode = PackageManager.INSTALL_SUCCEEDED;
		String errorMsg = null;
		ScanParams scanParams = result.scanParams;

		if (throwable == null) {
		try {
		Trace.traceBegin(TRACE_TAG_PACKAGE_MANAGER, "addForInitLI");
		addForInitLI(parseResult.parsedPackage, parseFlags, scanFlags,
		new UserHandle(UserHandle.USER_SYSTEM), apexInfo);
		addForInitLI(result.parsedPackage, scanParams.parseFlags, scanParams.scanFlags,
		new UserHandle(UserHandle.USER_SYSTEM), scanParams.apexInfo);
		} catch (PackageManagerException e) {
		errorCode = e.error;
		errorMsg = "Failed to scan " + parseResult.scanFile + ": " + e.getMessage();
		errorMsg = "Failed to scan " + result.scanFile + ": " + e.getMessage();
		Slog.w(TAG, errorMsg);
		} finally {
		Trace.traceEnd(TRACE_TAG_PACKAGE_MANAGER);
		@@ -3871,24 +3958,23 @@ final class InstallPackageHelper {
		} else if (throwable instanceof PackageManagerException) {
		PackageManagerException e = (PackageManagerException) throwable;
		errorCode = e.error;
		errorMsg = "Failed to parse " + parseResult.scanFile + ": " + e.getMessage();
		errorMsg = "Failed to parse " + result.scanFile + ": " + e.getMessage();
		Slog.w(TAG, errorMsg);
		} else {
		throw new IllegalStateException("Unexpected exception occurred while parsing "
		+ parseResult.scanFile, throwable);
		+ result.scanFile, throwable);
		}

		if ((scanFlags & SCAN_AS_APK_IN_APEX) != 0 && errorCode != INSTALL_SUCCEEDED) {
		mApexManager.reportErrorWithApkInApex(scanDir.getAbsolutePath(), errorMsg);
		if ((scanParams.scanFlags & SCAN_AS_APK_IN_APEX) != 0 && errorCode != INSTALL_SUCCEEDED) {
		mApexManager.reportErrorWithApkInApex(scanParams.scanDir.getAbsolutePath(), errorMsg);
		}

		// Delete invalid userdata apps
		if ((scanFlags & SCAN_AS_SYSTEM) == 0
		if ((scanParams.scanFlags & SCAN_AS_SYSTEM) == 0
		&& errorCode != PackageManager.INSTALL_SUCCEEDED) {
		logCriticalInfo(Log.WARN,
		"Deleting invalid package at " + parseResult.scanFile);
		mRemovePackageHelper.removeCodePath(parseResult.scanFile);
		}
		"Deleting invalid package at " + result.scanFile);
		mRemovePackageHelper.removeCodePath(result.scanFile);
		}
		}

services/core/java/com/android/server/pm/ParallelPackageParser.java

+41 −3

Original line number	Diff line number	Diff line
		@@ -18,6 +18,8 @@ package com.android.server.pm;

		import static android.os.Trace.TRACE_TAG_PACKAGE_MANAGER;

		import static com.android.server.pm.InitAppsHelper.ScanParams;

		import android.os.Process;
		import android.os.Trace;

		@@ -31,6 +33,7 @@ import java.io.File;
		import java.util.concurrent.ArrayBlockingQueue;
		import java.util.concurrent.BlockingQueue;
		import java.util.concurrent.ExecutorService;
		import java.util.concurrent.Future;

		/**
		* Helper class for parallel parsing of packages using {@link PackageParser2}.
		@@ -65,6 +68,7 @@ class ParallelPackageParser {
		ParsedPackage parsedPackage; // Parsed package
		File scanFile; // File that was parsed
		Throwable throwable; // Set if an error occurs during parsing
		ScanParams scanParams; // Parameters used when scanning and parsing this scanFile

		@Override
		public String toString() {
		@@ -97,15 +101,16 @@ class ParallelPackageParser {
		/**
		* Submits the file for parsing
		* @param scanFile file to scan
		* @param parseFlags parse flags
		* @param scanParams scan parameters
		*/
		public void submit(File scanFile, int parseFlags) {
		public void submit(File scanFile, ScanParams scanParams) {
		mExecutorService.submit(() -> {
		ParseResult pr = new ParseResult();
		Trace.traceBegin(TRACE_TAG_PACKAGE_MANAGER, "parallel parsePackage [" + scanFile + "]");
		try {
		pr.scanFile = scanFile;
		pr.parsedPackage = parsePackage(scanFile, parseFlags);
		pr.scanParams = scanParams;
		pr.parsedPackage = parsePackage(scanFile, scanParams.parseFlags);
		} catch (Throwable e) {
		pr.throwable = e;
		} finally {
		@@ -123,6 +128,39 @@ class ParallelPackageParser {
		});
		}

		static class OrderedResult {
		public final File scanFile; // The file that will be parsed. This is saved in case it needs
		// to be logged in an error.
		public final Future<ParseResult> future;
		OrderedResult(File scanFile, Future<ParseResult> future) {
		this.scanFile = scanFile;
		this.future = future;
		}
		}

		/**
		* Submits the file for parsing. The return result should be handled directly by the caller
		* instead of being pulled from take() later.
		* @param scanFile file to scan
		* @param scanParams scan parameters
		*/
		public OrderedResult orderedSubmit(File scanFile, ScanParams scanParams) {
		return new OrderedResult(scanFile, mExecutorService.submit(() -> {
		ParseResult pr = new ParseResult();
		Trace.traceBegin(TRACE_TAG_PACKAGE_MANAGER, "parallel parsePackage [" + scanFile + "]");
		try {
		pr.scanFile = scanFile;
		pr.scanParams = scanParams;
		pr.parsedPackage = parsePackage(scanFile, scanParams.parseFlags);
		} catch (Throwable e) {
		pr.throwable = e;
		} finally {
		Trace.traceEnd(TRACE_TAG_PACKAGE_MANAGER);
		}
		return pr;
		}));
		}

		@VisibleForTesting
		protected ParsedPackage parsePackage(File scanFile, int parseFlags)
		throws PackageManagerException {

services/tests/PackageManagerServiceTests/server/src/com/android/server/pm/ParallelPackageParserTest.java

+3 −1

Original line number	Diff line number	Diff line
		@@ -16,6 +16,8 @@

		package com.android.server.pm;

		import static com.android.server.pm.InitAppsHelper.ScanParams;

		import android.platform.test.annotations.Presubmit;
		import android.util.Log;

		@@ -58,7 +60,7 @@ public class ParallelPackageParserTest {
		int fileCount = 15;
		for (int i = 0; i < fileCount; i++) {
		File file = new File("f" + i);
		mParser.submit(file, 0);
		mParser.submit(file, ScanParams.forApkPartitionScan(null, 0, 0));
		submittedFiles.add(file);
		Log.d(TAG, "submitting " + file);
		}