Merge "Use module type, size of srcs or deps to prioritize module" (291cc5f9) · Commits · e / os / android_build_soong

android/allowlists/allowlists.go

+14 −263

Original line number	Diff line number	Diff line
		@@ -38,7 +38,13 @@ const (
		// allows modules to opt-in.
		Bp2BuildDefaultFalseRecursively

		DEFAULT_NINJA_WEIGHT = 1000
		// Modules with build time of more than half a minute should have high priority.
		DEFAULT_PRIORITIZED_WEIGHT = 1000
		// Modules with build time of more than a few minute should have higher priority.
		HIGH_PRIORITIZED_WEIGHT = 10 * DEFAULT_PRIORITIZED_WEIGHT
		// Modules with inputs greater than the threshold should have high priority.
		// Adjust this threshold if there are lots of wrong predictions.
		INPUT_SIZE_THRESHOLD = 50
		)

		var (
		@@ -1562,267 +1568,12 @@ var (
		// "libstagefright_flacdec",
		// "libutils",

		// TODO(b/273282046): Make this list customizable to support various targets.
		// The list of modules which are expected to spend lots of build time.
		// With `--ninja_weight_source=soong`, ninja builds these modules and deps first.
		HugeModulesMap = map[string]int{
		"AccountManagementApp": DEFAULT_NINJA_WEIGHT,
		"ActivityManagerPerfTestsStubApp1": DEFAULT_NINJA_WEIGHT,
		"ActivityManagerPerfTestsStubApp2": DEFAULT_NINJA_WEIGHT,
		"ActivityManagerPerfTestsStubApp3": DEFAULT_NINJA_WEIGHT,
		"api-stubs-docs-non-updatable": DEFAULT_NINJA_WEIGHT,
		"AppCompatibilityTest": DEFAULT_NINJA_WEIGHT,
		"AppTransitionTests": DEFAULT_NINJA_WEIGHT,
		"art_compiler_tests": DEFAULT_NINJA_WEIGHT,
		"art.module.intra.core.api.stubs.source": DEFAULT_NINJA_WEIGHT,
		"art.module.public.api.stubs.source": DEFAULT_NINJA_WEIGHT,
		"AttestationVerificationTest": DEFAULT_NINJA_WEIGHT,
		"BatteryUsageStatsProtoTests": DEFAULT_NINJA_WEIGHT,
		"bluetooth_test_gd_unit": DEFAULT_NINJA_WEIGHT,
		"Bluetooth": DEFAULT_NINJA_WEIGHT,
		"BluetoothInstrumentationTests": DEFAULT_NINJA_WEIGHT,
		"Calendar": DEFAULT_NINJA_WEIGHT,
		"CalendarProvider": DEFAULT_NINJA_WEIGHT,
		"Camera2": DEFAULT_NINJA_WEIGHT,
		"CarRotaryControllerUnitTests": DEFAULT_NINJA_WEIGHT,
		"CarSettingsForUnitTesting": DEFAULT_NINJA_WEIGHT,
		"CarSettingsUnitTests": DEFAULT_NINJA_WEIGHT,
		"CarSystemUI-tests": DEFAULT_NINJA_WEIGHT,
		"CellBroadcastApp": DEFAULT_NINJA_WEIGHT,
		"CellBroadcastLegacyApp": DEFAULT_NINJA_WEIGHT,
		"CellBroadcastReceiverOemUnitTests": DEFAULT_NINJA_WEIGHT,
		"CellBroadcastServiceModule": DEFAULT_NINJA_WEIGHT,
		"CompanionDeviceManager": DEFAULT_NINJA_WEIGHT,
		"ConnectivityChecker": DEFAULT_NINJA_WEIGHT,
		"Contacts": DEFAULT_NINJA_WEIGHT,
		"ContactsProvider": DEFAULT_NINJA_WEIGHT,
		"ContentCapturePerfTests": DEFAULT_NINJA_WEIGHT,
		"CorePerfTests": DEFAULT_NINJA_WEIGHT,
		"crosvm": DEFAULT_NINJA_WEIGHT,
		"CtsDomainVerificationDeviceMultiUserTestCases": DEFAULT_NINJA_WEIGHT,
		"CtsLogdTestCases": DEFAULT_NINJA_WEIGHT,
		"CtsMediaProviderTranscodeTests": DEFAULT_NINJA_WEIGHT,
		"CtsRollbackManagerHostTestHelperApp": DEFAULT_NINJA_WEIGHT,
		"CtsRollbackManagerHostTestHelperApp2": DEFAULT_NINJA_WEIGHT,
		"CtsRootPackageInstallerTestCases": DEFAULT_NINJA_WEIGHT,
		"CtsRootRollbackManagerHostTestHelperApp": DEFAULT_NINJA_WEIGHT,
		"CtsTranscodeTestAppSupportsHevc": DEFAULT_NINJA_WEIGHT,
		"CtsTranscodeTestAppSupportsSlowMotion": DEFAULT_NINJA_WEIGHT,
		"CuttlefishDisplayHotplugHelperApp": DEFAULT_NINJA_WEIGHT,
		"cvd-host_package": DEFAULT_NINJA_WEIGHT,
		"DelegateTestApp": DEFAULT_NINJA_WEIGHT,
		"DeskClock": DEFAULT_NINJA_WEIGHT,
		"Development": DEFAULT_NINJA_WEIGHT,
		"DeviceAdminTestApp": DEFAULT_NINJA_WEIGHT,
		"DevicePolicyManagementRoleHolderTestApp": DEFAULT_NINJA_WEIGHT,
		"dex2oatd": DEFAULT_NINJA_WEIGHT,
		"DocumentsUI": DEFAULT_NINJA_WEIGHT,
		"EasterEgg": DEFAULT_NINJA_WEIGHT,
		"EffectProxyTest": DEFAULT_NINJA_WEIGHT,
		"EmergencyInfo": DEFAULT_NINJA_WEIGHT,
		"EmptyTestApp": DEFAULT_NINJA_WEIGHT,
		"ExtServices": DEFAULT_NINJA_WEIGHT,
		"FacebookAppsScenarioTests": DEFAULT_NINJA_WEIGHT,
		"flickerlib-core": DEFAULT_NINJA_WEIGHT,
		"flickerlib": DEFAULT_NINJA_WEIGHT,
		"FlickerLibTest": DEFAULT_NINJA_WEIGHT,
		"FlickerTests": DEFAULT_NINJA_WEIGHT,
		"framework-minus-apex": DEFAULT_NINJA_WEIGHT,
		"framework-res": DEFAULT_NINJA_WEIGHT,
		"FrameworksCoreTests": DEFAULT_NINJA_WEIGHT,
		"FrameworksMockingCoreTests": DEFAULT_NINJA_WEIGHT,
		"FrameworksMockingServicesTests": DEFAULT_NINJA_WEIGHT,
		"FrameworksNetSmokeTests": DEFAULT_NINJA_WEIGHT,
		"FrameworksNetTests": DEFAULT_NINJA_WEIGHT,
		"FrameworksServicesTests": DEFAULT_NINJA_WEIGHT,
		"FrameworksTelephonyTests": DEFAULT_NINJA_WEIGHT,
		"FrameworksUiServicesTests": DEFAULT_NINJA_WEIGHT,
		"FrameworksVcnTests": DEFAULT_NINJA_WEIGHT,
		"Gallery2": DEFAULT_NINJA_WEIGHT,
		"GameCoreDevice": DEFAULT_NINJA_WEIGHT,
		"GoogleBluetoothInstrumentationTests": DEFAULT_NINJA_WEIGHT,
		"guice_munged_srcs": DEFAULT_NINJA_WEIGHT,
		"HalfSheetUX": DEFAULT_NINJA_WEIGHT,
		"ImePerfTests": DEFAULT_NINJA_WEIGHT,
		"imgdiag": DEFAULT_NINJA_WEIGHT,
		"ImsServiceEntitlement": DEFAULT_NINJA_WEIGHT,
		"ImsServiceEntitlementUnitTests": DEFAULT_NINJA_WEIGHT,
		"InputTests": DEFAULT_NINJA_WEIGHT,
		"InstallTest": DEFAULT_NINJA_WEIGHT,
		"IntentResolver": DEFAULT_NINJA_WEIGHT,
		"JankBench": DEFAULT_NINJA_WEIGHT,
		"jsilver": DEFAULT_NINJA_WEIGHT,
		"KeyChain": DEFAULT_NINJA_WEIGHT,
		"KeyChainTests": DEFAULT_NINJA_WEIGHT,
		"keystore2": DEFAULT_NINJA_WEIGHT,
		"LargeResourcesCompressed": DEFAULT_NINJA_WEIGHT,
		"LatinIME": DEFAULT_NINJA_WEIGHT,
		"Launcher3QuickStepLib": DEFAULT_NINJA_WEIGHT,
		"libaom": DEFAULT_NINJA_WEIGHT,
		"libart-broken": DEFAULT_NINJA_WEIGHT,
		"libart-compiler": DEFAULT_NINJA_WEIGHT,
		"libart-disassembler": DEFAULT_NINJA_WEIGHT,
		"libart": DEFAULT_NINJA_WEIGHT,
		"libartd": DEFAULT_NINJA_WEIGHT,
		"libaudiohal@7.1": DEFAULT_NINJA_WEIGHT,
		"libbluetooth_core_rs": DEFAULT_NINJA_WEIGHT,
		"libbluetooth_gd_unit_tests": DEFAULT_NINJA_WEIGHT,
		"libbluetooth_gd": DEFAULT_NINJA_WEIGHT,
		"libbluetooth_rs": DEFAULT_NINJA_WEIGHT,
		"libbluetooth-for-tests": DEFAULT_NINJA_WEIGHT,
		"libbt_common": DEFAULT_NINJA_WEIGHT,
		"libbt_packets_nonapex": DEFAULT_NINJA_WEIGHT,
		"libbt_packets": DEFAULT_NINJA_WEIGHT,
		"libbt_shim_ffi": DEFAULT_NINJA_WEIGHT,
		"libbt_shim": DEFAULT_NINJA_WEIGHT,
		"libbt-audio-hal-interface": DEFAULT_NINJA_WEIGHT,
		"libbt-bta-core": DEFAULT_NINJA_WEIGHT,
		"libbt-bta": DEFAULT_NINJA_WEIGHT,
		"libbt-common": DEFAULT_NINJA_WEIGHT,
		"libbt-hci": DEFAULT_NINJA_WEIGHT,
		"libbt-platform-protos-lite": DEFAULT_NINJA_WEIGHT,
		"libbt-protos-lite": DEFAULT_NINJA_WEIGHT,
		"libbt-sbc-decoder": DEFAULT_NINJA_WEIGHT,
		"libc": DEFAULT_NINJA_WEIGHT,
		"libclap": DEFAULT_NINJA_WEIGHT,
		"libcodec2_soft_av1dec_gav1": DEFAULT_NINJA_WEIGHT,
		"libcompositionengine_test": DEFAULT_NINJA_WEIGHT,
		"libdevices": DEFAULT_NINJA_WEIGHT,
		"libfrontend_proto": DEFAULT_NINJA_WEIGHT,
		"libhwtrust": DEFAULT_NINJA_WEIGHT,
		"libjni": DEFAULT_NINJA_WEIGHT,
		"libkeystore2": DEFAULT_NINJA_WEIGHT,
		"libkmr_ta": DEFAULT_NINJA_WEIGHT,
		"liblmp": DEFAULT_NINJA_WEIGHT,
		"libopenjdkjvmtid": DEFAULT_NINJA_WEIGHT,
		"libprotobuf_deprecated": DEFAULT_NINJA_WEIGHT,
		"libprotobuf": DEFAULT_NINJA_WEIGHT,
		"libregex": DEFAULT_NINJA_WEIGHT,
		"LibStatsPullTests": DEFAULT_NINJA_WEIGHT,
		"libstd": DEFAULT_NINJA_WEIGHT,
		"libsurfaceflinger_unittest": DEFAULT_NINJA_WEIGHT,
		"libsyn": DEFAULT_NINJA_WEIGHT,
		"libtokio": DEFAULT_NINJA_WEIGHT,
		"libuwb_core": DEFAULT_NINJA_WEIGHT,
		"libuwb_uci_jni_rust": DEFAULT_NINJA_WEIGHT,
		"libuwb_uci_packets": DEFAULT_NINJA_WEIGHT,
		"libvpx": DEFAULT_NINJA_WEIGHT,
		"libvulkan_enc": DEFAULT_NINJA_WEIGHT,
		"libwebrtc": DEFAULT_NINJA_WEIGHT,
		"LiveWallpapersPicker": DEFAULT_NINJA_WEIGHT,
		"LockTaskApp": DEFAULT_NINJA_WEIGHT,
		"LongevityPlatformLibTests": DEFAULT_NINJA_WEIGHT,
		"ManagedProvisioning": DEFAULT_NINJA_WEIGHT,
		"ManagedProvisioningTests": DEFAULT_NINJA_WEIGHT,
		"MediaProvider": DEFAULT_NINJA_WEIGHT,
		"MediaProviderClientTests": DEFAULT_NINJA_WEIGHT,
		"MediaProviderLegacy": DEFAULT_NINJA_WEIGHT,
		"messaging": DEFAULT_NINJA_WEIGHT,
		"metalava": DEFAULT_NINJA_WEIGHT,
		"MicrobenchmarkRunnerTests": DEFAULT_NINJA_WEIGHT,
		"microdroid_manager": DEFAULT_NINJA_WEIGHT,
		"minikin_tests": DEFAULT_NINJA_WEIGHT,
		"MLCTestApp": DEFAULT_NINJA_WEIGHT,
		"MmsService": DEFAULT_NINJA_WEIGHT,
		"MmsServiceTests": DEFAULT_NINJA_WEIGHT,
		"module-lib-api-stubs-docs-non-updatable": DEFAULT_NINJA_WEIGHT,
		"motion_tool_lib_tests": DEFAULT_NINJA_WEIGHT,
		"MtpService": DEFAULT_NINJA_WEIGHT,
		"MultiUserTests": DEFAULT_NINJA_WEIGHT,
		"NearbyIntegrationUiTests": DEFAULT_NINJA_WEIGHT,
		"net_test_bluetooth": DEFAULT_NINJA_WEIGHT,
		"net_test_btif": DEFAULT_NINJA_WEIGHT,
		"net_test_main_shim": DEFAULT_NINJA_WEIGHT,
		"net_test_stack": DEFAULT_NINJA_WEIGHT,
		"net-tests-utils": DEFAULT_NINJA_WEIGHT,
		"NetworkStackCoverageTests": DEFAULT_NINJA_WEIGHT,
		"NetworkStackIntegrationTests": DEFAULT_NINJA_WEIGHT,
		"NetworkStackNext": DEFAULT_NINJA_WEIGHT,
		"NfcNci": DEFAULT_NINJA_WEIGHT,
		"NfcNciUnitTests": DEFAULT_NINJA_WEIGHT,
		"NotEmptyTestApp": DEFAULT_NINJA_WEIGHT,
		"NotificationFunctionalTests": DEFAULT_NINJA_WEIGHT,
		"oatdumpd": DEFAULT_NINJA_WEIGHT,
		"OsuLogin": DEFAULT_NINJA_WEIGHT,
		"PackageInstaller": DEFAULT_NINJA_WEIGHT,
		"PackageManagerComponentOverrideTests": DEFAULT_NINJA_WEIGHT,
		"PackageManagerPerfTests": DEFAULT_NINJA_WEIGHT,
		"PackageManagerServiceServerTests": DEFAULT_NINJA_WEIGHT,
		"PackageManagerServiceUnitTests": DEFAULT_NINJA_WEIGHT,
		"PackageWatchdogTest": DEFAULT_NINJA_WEIGHT,
		"PandoraServerLib": DEFAULT_NINJA_WEIGHT,
		"pdl": DEFAULT_NINJA_WEIGHT,
		"perfetto_trace_java_protos": DEFAULT_NINJA_WEIGHT,
		"perfetto_trace-full": DEFAULT_NINJA_WEIGHT,
		"PermissionController": DEFAULT_NINJA_WEIGHT,
		"PermissionControllerMockingTests": DEFAULT_NINJA_WEIGHT,
		"PixelAppCompTests": DEFAULT_NINJA_WEIGHT,
		"platform-bootclasspath": DEFAULT_NINJA_WEIGHT,
		"PlatformCommonScenarioTests": DEFAULT_NINJA_WEIGHT,
		"PlatformComposeCoreTests": DEFAULT_NINJA_WEIGHT,
		"platformprotoslite": DEFAULT_NINJA_WEIGHT,
		"PlatformRuleTests": DEFAULT_NINJA_WEIGHT,
		"precompiled_sepolicy-without_apex": DEFAULT_NINJA_WEIGHT,
		"PresencePolling": DEFAULT_NINJA_WEIGHT,
		"PrintSpooler": DEFAULT_NINJA_WEIGHT,
		"QuickSearchBox": DEFAULT_NINJA_WEIGHT,
		"RemoteDPCTestApp": DEFAULT_NINJA_WEIGHT,
		"RemoteProvisioningServiceTests": DEFAULT_NINJA_WEIGHT,
		"RkpdAppUnitTests": DEFAULT_NINJA_WEIGHT,
		"Robolectric_shadows_framework": DEFAULT_NINJA_WEIGHT,
		"RoleHolderApp": DEFAULT_NINJA_WEIGHT,
		"SdkSandbox": DEFAULT_NINJA_WEIGHT,
		"service-appsearch": DEFAULT_NINJA_WEIGHT,
		"service-connectivity": DEFAULT_NINJA_WEIGHT,
		"service-uwb": DEFAULT_NINJA_WEIGHT,
		"service-wifi": DEFAULT_NINJA_WEIGHT,
		"services-non-updatable-stubs": DEFAULT_NINJA_WEIGHT,
		"services": DEFAULT_NINJA_WEIGHT,
		"Settings-core": DEFAULT_NINJA_WEIGHT,
		"Settings": DEFAULT_NINJA_WEIGHT,
		"SettingsIntelligence": DEFAULT_NINJA_WEIGHT,
		"SettingsLibTests": DEFAULT_NINJA_WEIGHT,
		"SettingsProvider": DEFAULT_NINJA_WEIGHT,
		"Shell": DEFAULT_NINJA_WEIGHT,
		"SimAppDialog": DEFAULT_NINJA_WEIGHT,
		"sl4a": DEFAULT_NINJA_WEIGHT,
		"SmsApp": DEFAULT_NINJA_WEIGHT,
		"SoundPicker": DEFAULT_NINJA_WEIGHT,
		"StagedInstallTest": DEFAULT_NINJA_WEIGHT,
		"StatementService": DEFAULT_NINJA_WEIGHT,
		"StatsdFrameworkTestApp": DEFAULT_NINJA_WEIGHT,
		"StatsdFrameworkTestAppNoPermission": DEFAULT_NINJA_WEIGHT,
		"statsdprotolite": DEFAULT_NINJA_WEIGHT,
		"Stk": DEFAULT_NINJA_WEIGHT,
		"StorageManager": DEFAULT_NINJA_WEIGHT,
		"system-api-stubs-docs-non-updatable": DEFAULT_NINJA_WEIGHT,
		"SystemUI-core": DEFAULT_NINJA_WEIGHT,
		"SystemUI-tests-base": DEFAULT_NINJA_WEIGHT,
		"SystemUI-tests": DEFAULT_NINJA_WEIGHT,
		"SystemUI": DEFAULT_NINJA_WEIGHT,
		"SystemUIComposeFeatures": DEFAULT_NINJA_WEIGHT,
		"SystemUIComposeFeaturesTests": DEFAULT_NINJA_WEIGHT,
		"SystemUITests": DEFAULT_NINJA_WEIGHT,
		"Tag": DEFAULT_NINJA_WEIGHT,
		"Telecom": DEFAULT_NINJA_WEIGHT,
		"TelecomUnitTests": DEFAULT_NINJA_WEIGHT,
		"telephony-common": DEFAULT_NINJA_WEIGHT,
		"TelephonyProvider": DEFAULT_NINJA_WEIGHT,
		"TeleService": DEFAULT_NINJA_WEIGHT,
		"test-api-stubs-docs-non-updatable": DEFAULT_NINJA_WEIGHT,
		"TetheringIntegrationTests": DEFAULT_NINJA_WEIGHT,
		"TetheringNext": DEFAULT_NINJA_WEIGHT,
		"ThemePickerTests": DEFAULT_NINJA_WEIGHT,
		"Traceur": DEFAULT_NINJA_WEIGHT,
		"UsbManagerTests": DEFAULT_NINJA_WEIGHT,
		"UsbTests": DEFAULT_NINJA_WEIGHT,
		"virtmgr": DEFAULT_NINJA_WEIGHT,
		"WallpaperPicker2TestLib": DEFAULT_NINJA_WEIGHT,
		"WallpaperPicker2Tests": DEFAULT_NINJA_WEIGHT,
		"WifiDialog": DEFAULT_NINJA_WEIGHT,
		"wm-proto-parsers": DEFAULT_NINJA_WEIGHT,
		"WMShellFlickerTests": DEFAULT_NINJA_WEIGHT,
		"WmTests": DEFAULT_NINJA_WEIGHT,
		"wpa_supplicant": DEFAULT_NINJA_WEIGHT,
		// The list of module types which are expected to spend lots of build time.
		// With `--ninja_weight_source=soong`, ninja builds these module types and deps first.
		HugeModuleTypePrefixMap = map[string]int{
		"rust_": HIGH_PRIORITIZED_WEIGHT,
		"droidstubs": DEFAULT_PRIORITIZED_WEIGHT,
		"art_": DEFAULT_PRIORITIZED_WEIGHT,
		"ndk_library": DEFAULT_PRIORITIZED_WEIGHT,
		}
		)

cmd/soong_build/main.go

+38 −10

Original line number	Diff line number	Diff line
		@@ -30,6 +30,7 @@ import (
		"android/soong/shared"
		"android/soong/ui/metrics/bp2build_metrics_proto"

		"github.com/google/blueprint"
		"github.com/google/blueprint/bootstrap"
		"github.com/google/blueprint/deptools"
		"github.com/google/blueprint/metrics"
		@@ -256,18 +257,45 @@ func apiBuildFileExcludes(ctx *android.Context) []string {
		}

		func writeNinjaHint(ctx *android.Context) error {
		wantModules := make([]string, len(allowlists.HugeModulesMap))
		i := 0
		for k := range allowlists.HugeModulesMap {
		wantModules[i] = k
		i += 1
		// The current predictor focuses on reducing false negatives.
		// If there are too many false positives (e.g., most modules are marked as positive),
		// real long-running jobs cannot run early.
		// Therefore, the model should be adjusted in this case.
		// The model should also be adjusted if there are critical false negatives.
		predicate := func(j *blueprint.JsonModule) (prioritized bool, weight int) {
		prioritized = false
		weight = 0
		for prefix, w := range allowlists.HugeModuleTypePrefixMap {
		if strings.HasPrefix(j.Type, prefix) {
		prioritized = true
		weight = w
		return
		}
		outputsMap := ctx.Context.GetOutputsFromModuleNames(wantModules)
		var outputBuilder strings.Builder
		for k, v := range allowlists.HugeModulesMap {
		for _, output := range outputsMap[k] {
		outputBuilder.WriteString(fmt.Sprintf("%s,%d\n", output, v))
		}
		dep_count := len(j.Deps)
		src_count := 0
		for _, a := range j.Module["Actions"].([]blueprint.JSONAction) {
		src_count += len(a.Inputs)
		}
		input_size := dep_count + src_count

		// Current threshold is an arbitrary value which only consider recall rather than accuracy.
		if input_size > allowlists.INPUT_SIZE_THRESHOLD {
		prioritized = true
		weight += ((input_size) / allowlists.INPUT_SIZE_THRESHOLD) * allowlists.DEFAULT_PRIORITIZED_WEIGHT

		// To prevent some modules from having too large a priority value.
		if weight > allowlists.HIGH_PRIORITIZED_WEIGHT {
		weight = allowlists.HIGH_PRIORITIZED_WEIGHT
		}
		}
		return
		}

		outputsMap := ctx.Context.GetWeightedOutputsFromPredicate(predicate)
		var outputBuilder strings.Builder
		for output, weight := range outputsMap {
		outputBuilder.WriteString(fmt.Sprintf("%s,%d\n", output, weight))
		}
		weightListFile := filepath.Join(topDir, ctx.Config().OutDir(), ".ninja_weight_list")