Merge "Allow running bp2build as part of a regular build." (3d76d5d6) · Commits · e / os / android_build_soong

cmd/soong_build/main.go

+97 −44

Original line number	Diff line number	Diff line
		@@ -23,29 +23,42 @@ import (
		"strings"
		"time"

		"android/soong/bp2build"
		"android/soong/shared"
		"github.com/google/blueprint/bootstrap"

		"android/soong/android"
		"android/soong/bp2build"
		)

		var (
		topDir string
		outDir string
		docFile string
		bazelQueryViewDir string
		availableEnvFile string
		usedEnvFile string

		delveListen string
		delvePath string

		docFile string
		bazelQueryViewDir string
		bp2buildMarker string
		)

		func init() {
		// Flags that make sense in every mode
		flag.StringVar(&topDir, "top", "", "Top directory of the Android source tree")
		flag.StringVar(&outDir, "out", "", "Soong output directory (usually $TOP/out/soong)")
		flag.StringVar(&availableEnvFile, "available_env", "", "File containing available environment variables")
		flag.StringVar(&usedEnvFile, "used_env", "", "File containing used environment variables")

		// Debug flags
		flag.StringVar(&delveListen, "delve_listen", "", "Delve port to listen on for debugging")
		flag.StringVar(&delvePath, "delve_path", "", "Path to Delve. Only used if --delve_listen is set")

		// Flags representing various modes soong_build can run in
		flag.StringVar(&docFile, "soong_docs", "", "build documentation file to output")
		flag.StringVar(&bazelQueryViewDir, "bazel_queryview_dir", "", "path to the bazel queryview directory relative to --top")
		flag.StringVar(&bp2buildMarker, "bp2build_marker", "", "If set, run bp2build, touch the specified marker file then exit")
		}

		func newNameResolver(config android.Config) *android.NameResolver {
		@@ -147,8 +160,8 @@ func writeJsonModuleGraph(configuration android.Config, ctx *android.Context, pa
		writeFakeNinjaFile(extraNinjaDeps, configuration.BuildDir())
		}

		func doChosenActivity(configuration android.Config, extraNinjaDeps []string) {
		bazelConversionRequested := configuration.IsEnvTrue("GENERATE_BAZEL_FILES")
		func doChosenActivity(configuration android.Config, extraNinjaDeps []string) string {
		bazelConversionRequested := configuration.IsEnvTrue("GENERATE_BAZEL_FILES") \|\| bp2buildMarker != ""
		mixedModeBuild := configuration.BazelContext.BazelEnabled()
		generateQueryView := bazelQueryViewDir != ""
		jsonModuleFile := configuration.Getenv("SOONG_DUMP_JSON_MODULE_GRAPH")
		@@ -159,7 +172,11 @@ func doChosenActivity(configuration android.Config, extraNinjaDeps []string) {
		// Run the alternate pipeline of bp2build mutators and singleton to convert
		// Blueprint to BUILD files before everything else.
		runBp2Build(configuration, extraNinjaDeps)
		return
		if bp2buildMarker != "" {
		return bp2buildMarker
		} else {
		return bootstrap.CmdlineOutFile()
		}
		}

		ctx := newContext(configuration, prepareBuildActions)
		@@ -172,23 +189,18 @@ func doChosenActivity(configuration android.Config, extraNinjaDeps []string) {
		// Convert the Soong module graph into Bazel BUILD files.
		if generateQueryView {
		runQueryView(configuration, ctx)
		return
		return bootstrap.CmdlineOutFile() // TODO: This is a lie
		}

		if jsonModuleFile != "" {
		writeJsonModuleGraph(configuration, ctx, jsonModuleFile, extraNinjaDeps)
		return
		return bootstrap.CmdlineOutFile() // TODO: This is a lie
		}

		writeMetrics(configuration)
		return bootstrap.CmdlineOutFile()
		}

		func main() {
		flag.Parse()

		shared.ReexecWithDelveMaybe(delveListen, delvePath)
		android.InitSandbox(topDir)

		// soong_ui dumps the available environment variables to
		// soong.environment.available . Then soong_build itself is run with an empty
		// environment so that the only way environment variables can be accessed is
		@@ -202,14 +214,29 @@ func main() {
		//
		// The dependency of build.ninja on soong.environment.used is declared in
		// build.ninja.d
		availableEnvFile := shared.JoinPath(topDir, outDir, "soong.environment.available")
		usedEnvFile := shared.JoinPath(topDir, outDir, "soong.environment.used")
		availableEnv, err := shared.EnvFromFile(availableEnvFile)
		func parseAvailableEnv() map[string]string {
		if availableEnvFile == "" {
		fmt.Fprintf(os.Stderr, "--available_env not set\n")
		os.Exit(1)
		}

		result, err := shared.EnvFromFile(shared.JoinPath(topDir, availableEnvFile))
		if err != nil {
		fmt.Fprintf(os.Stderr, "error reading available environment file %s: %s\n", availableEnvFile, err)
		fmt.Fprintf(os.Stderr, "error reading available environment file '%s': %s\n", availableEnvFile, err)
		os.Exit(1)
		}

		return result
		}

		func main() {
		flag.Parse()

		shared.ReexecWithDelveMaybe(delveListen, delvePath)
		android.InitSandbox(topDir)

		availableEnv := parseAvailableEnv()

		// The top-level Blueprints file is passed as the first argument.
		srcDir := filepath.Dir(flag.Arg(0))
		configuration := newConfig(srcDir, outDir, availableEnv)
		@@ -233,37 +260,37 @@ func main() {
		// because that is done from within the actual builds as a Ninja action and
		// thus it would overwrite the actual used variables file so this is
		// special-cased.
		// TODO: Fix this by not passing --used_env to the soong_docs invocation
		runSoongDocs(configuration, extraNinjaDeps)
		return
		}

		doChosenActivity(configuration, extraNinjaDeps)
		writeUsedEnvironmentFile(usedEnvFile, configuration)
		finalOutputFile := doChosenActivity(configuration, extraNinjaDeps)
		writeUsedEnvironmentFile(configuration, finalOutputFile)
		}

		func writeUsedEnvironmentFile(configuration android.Config, finalOutputFile string) {
		if usedEnvFile == "" {
		return
		}

		func writeUsedEnvironmentFile(path string, configuration android.Config) {
		path := shared.JoinPath(topDir, usedEnvFile)
		data, err := shared.EnvFileContents(configuration.EnvDeps())
		if err != nil {
		fmt.Fprintf(os.Stderr, "error writing used environment file %s: %s\n", path, err)
		fmt.Fprintf(os.Stderr, "error writing used environment file '%s': %s\n", usedEnvFile, err)
		os.Exit(1)
		}

		err = ioutil.WriteFile(path, data, 0666)
		if err != nil {
		fmt.Fprintf(os.Stderr, "error writing used environment file %s: %s\n", path, err)
		fmt.Fprintf(os.Stderr, "error writing used environment file '%s': %s\n", usedEnvFile, err)
		os.Exit(1)
		}

		// Touch the output Ninja file so that it's not older than the file we just
		// Touch the output file so that it's not older than the file we just
		// wrote. We can't write the environment file earlier because one an access
		// new environment variables while writing it.
		outputNinjaFile := shared.JoinPath(topDir, bootstrap.CmdlineOutFile())
		currentTime := time.Now().Local()
		err = os.Chtimes(outputNinjaFile, currentTime, currentTime)
		if err != nil {
		fmt.Fprintf(os.Stderr, "error touching output file %s: %s\n", outputNinjaFile, err)
		os.Exit(1)
		}
		touch(shared.JoinPath(topDir, finalOutputFile))
		}

		// Workarounds to support running bp2build in a clean AOSP checkout with no
		@@ -289,6 +316,27 @@ func writeFakeNinjaFile(extraNinjaDeps []string, buildDir string) {
		0666)
		}

		func touch(path string) {
		f, err := os.OpenFile(path, os.O_APPEND\|os.O_CREATE\|os.O_WRONLY, 0666)
		if err != nil {
		fmt.Fprintf(os.Stderr, "Error touching '%s': %s\n", path, err)
		os.Exit(1)
		}

		err = f.Close()
		if err != nil {
		fmt.Fprintf(os.Stderr, "Error touching '%s': %s\n", path, err)
		os.Exit(1)
		}

		currentTime := time.Now().Local()
		err = os.Chtimes(path, currentTime, currentTime)
		if err != nil {
		fmt.Fprintf(os.Stderr, "error touching '%s': %s\n", path, err)
		os.Exit(1)
		}
		}

		// Run Soong in the bp2build mode. This creates a standalone context that registers
		// an alternate pipeline of mutators and singletons specifically for generating
		// Bazel BUILD files instead of Ninja files.
		@@ -296,6 +344,7 @@ func runBp2Build(configuration android.Config, extraNinjaDeps []string) {
		// Register an alternate set of singletons and mutators for bazel
		// conversion for Bazel conversion.
		bp2buildCtx := android.NewContext(configuration)
		bp2buildCtx.SetAllowMissingDependencies(configuration.AllowMissingDependencies())
		bp2buildCtx.RegisterForBazelConversion()

		// No need to generate Ninja build rules/statements from Modules and Singletons.
		@@ -330,5 +379,9 @@ func runBp2Build(configuration android.Config, extraNinjaDeps []string) {
		metrics.Print()

		extraNinjaDeps = append(extraNinjaDeps, codegenContext.AdditionalNinjaDeps()...)
		if bp2buildMarker != "" {
		touch(shared.JoinPath(topDir, bp2buildMarker))
		} else {
		writeFakeNinjaFile(extraNinjaDeps, codegenContext.Config().BuildDir())
		}
		}

tests/bootstrap_test.sh

+29 −2

Original line number	Diff line number	Diff line
		@@ -114,7 +114,9 @@ EOF
		rm a/Android.bp
		run_soong

		grep -q "^# Module:.*my_little_binary_host$" out/soong/build.ninja && fail "Old module in output"
		if grep -q "^# Module:.*my_little_binary_host$" out/soong/build.ninja; then
		fail "Old module in output"
		fi
		}

		function test_add_file_to_glob() {
		@@ -404,7 +406,9 @@ EOF

		grep -q "Engage" out/soong/build.ninja \|\| fail "New action not present"

		grep -q "Make it so" out/soong/build.ninja && fail "Original action still present"
		if grep -q "Make it so" out/soong/build.ninja; then
		fail "Original action still present"
		fi
		}

		function test_null_build_after_docs {
		@@ -421,6 +425,27 @@ function test_null_build_after_docs {
		fi
		}

		function test_integrated_bp2build_smoke {
		setup
		INTEGRATED_BP2BUILD=1 run_soong
		if [[ ! -e out/soong/.bootstrap/bp2build_workspace_marker ]]; then
		fail "b2build marker file not created"
		fi
		}

		function test_integrated_bp2build_null_build {
		setup
		INTEGRATED_BP2BUILD=1 run_soong
		local mtime1=$(stat -c "%y" out/soong/build.ninja)

		INTEGRATED_BP2BUILD=1 run_soong
		local mtime2=$(stat -c "%y" out/soong/build.ninja)

		if [[ "$mtime1" != "$mtime2" ]]; then
		fail "Output Ninja file changed on null build"
		fi
		}

		function test_dump_json_module_graph() {
		setup
		SOONG_DUMP_JSON_MODULE_GRAPH="$MOCK_TOP/modules.json" run_soong
		@@ -441,3 +466,5 @@ test_add_file_to_soong_build
		test_glob_during_bootstrapping
		test_soong_build_rerun_iff_environment_changes
		test_dump_json_module_graph
		test_integrated_bp2build_smoke
		test_integrated_bp2build_null_build

ui/build/soong.go

+75 −11

Original line number	Diff line number	Diff line
		@@ -33,6 +33,11 @@ import (
		"android/soong/ui/status"
		)

		const (
		availableEnvFile = "soong.environment.available"
		usedEnvFile = "soong.environment.used"
		)

		func writeEnvironmentFile(ctx Context, envFile string, envDeps map[string]string) error {
		data, err := shared.EnvFileContents(envDeps)
		if err != nil {
		@@ -87,12 +92,22 @@ func (c BlueprintConfig) DebugCompilation() bool {
		return c.debugCompilation
		}

		func bootstrapBlueprint(ctx Context, config Config) {
		func environmentArgs(config Config, suffix string) []string {
		return []string{
		"--available_env", shared.JoinPath(config.SoongOutDir(), availableEnvFile),
		"--used_env", shared.JoinPath(config.SoongOutDir(), usedEnvFile+suffix),
		}
		}
		func bootstrapBlueprint(ctx Context, config Config, integratedBp2Build bool) {
		ctx.BeginTrace(metrics.RunSoong, "blueprint bootstrap")
		defer ctx.EndTrace()

		var args bootstrap.Args

		mainNinjaFile := shared.JoinPath(config.SoongOutDir(), "build.ninja")
		globFile := shared.JoinPath(config.SoongOutDir(), ".bootstrap/soong-build-globs.ninja")
		bootstrapGlobFile := shared.JoinPath(config.SoongOutDir(), ".bootstrap/build-globs.ninja")

		args.RunGoTests = !config.skipSoongTests
		args.UseValidations = true // Use validations to depend on tests
		args.BuildDir = config.SoongOutDir()
		@@ -101,7 +116,7 @@ func bootstrapBlueprint(ctx Context, config Config) {
		args.ModuleListFile = filepath.Join(config.FileListDir(), "Android.bp.list")
		args.OutFile = shared.JoinPath(config.SoongOutDir(), ".bootstrap/build.ninja")
		args.DepFile = shared.JoinPath(config.SoongOutDir(), ".bootstrap/build.ninja.d")
		args.GlobFile = shared.JoinPath(config.SoongOutDir(), ".bootstrap/soong-build-globs.ninja")
		args.GlobFile = globFile
		args.GeneratingPrimaryBuilder = true

		args.DelveListen = os.Getenv("SOONG_DELVE")
		@@ -109,6 +124,44 @@ func bootstrapBlueprint(ctx Context, config Config) {
		args.DelvePath = shared.ResolveDelveBinary()
		}

		commonArgs := bootstrap.PrimaryBuilderExtraFlags(args, bootstrapGlobFile, mainNinjaFile)
		bp2BuildMarkerFile := shared.JoinPath(config.SoongOutDir(), ".bootstrap/bp2build_workspace_marker")
		mainSoongBuildInputs := []string{"Android.bp"}

		if integratedBp2Build {
		mainSoongBuildInputs = append(mainSoongBuildInputs, bp2BuildMarkerFile)
		}

		soongBuildArgs := make([]string, 0)
		soongBuildArgs = append(soongBuildArgs, commonArgs...)
		soongBuildArgs = append(soongBuildArgs, environmentArgs(config, "")...)
		soongBuildArgs = append(soongBuildArgs, "Android.bp")

		mainSoongBuildInvocation := bootstrap.PrimaryBuilderInvocation{
		Inputs: mainSoongBuildInputs,
		Outputs: []string{mainNinjaFile},
		Args: soongBuildArgs,
		}

		if integratedBp2Build {
		bp2buildArgs := []string{"--bp2build_marker", bp2BuildMarkerFile}
		bp2buildArgs = append(bp2buildArgs, commonArgs...)
		bp2buildArgs = append(bp2buildArgs, environmentArgs(config, ".bp2build")...)
		bp2buildArgs = append(bp2buildArgs, "Android.bp")

		bp2buildInvocation := bootstrap.PrimaryBuilderInvocation{
		Inputs: []string{"Android.bp"},
		Outputs: []string{bp2BuildMarkerFile},
		Args: bp2buildArgs,
		}
		args.PrimaryBuilderInvocations = []bootstrap.PrimaryBuilderInvocation{
		bp2buildInvocation,
		mainSoongBuildInvocation,
		}
		} else {
		args.PrimaryBuilderInvocations = []bootstrap.PrimaryBuilderInvocation{mainSoongBuildInvocation}
		}

		blueprintCtx := blueprint.NewContext()
		blueprintCtx.SetIgnoreUnknownModuleTypes(true)
		blueprintConfig := BlueprintConfig{
		@@ -121,6 +174,16 @@ func bootstrapBlueprint(ctx Context, config Config) {
		bootstrap.RunBlueprint(args, blueprintCtx, blueprintConfig)
		}

		func checkEnvironmentFile(currentEnv *Environment, envFile string) {
		getenv := func(k string) string {
		v, _ := currentEnv.Get(k)
		return v
		}
		if stale, _ := shared.StaleEnvFile(envFile, getenv); stale {
		os.Remove(envFile)
		}
		}

		func runSoong(ctx Context, config Config) {
		ctx.BeginTrace(metrics.RunSoong, "soong")
		defer ctx.EndTrace()
		@@ -129,7 +192,7 @@ func runSoong(ctx Context, config Config) {
		// .used with the ones that were actually used. The latter is used to
		// determine whether Soong needs to be re-run since why re-run it if only
		// unused variables were changed?
		envFile := filepath.Join(config.SoongOutDir(), "soong.environment.available")
		envFile := filepath.Join(config.SoongOutDir(), availableEnvFile)

		for _, n := range []string{".bootstrap", ".minibootstrap"} {
		dir := filepath.Join(config.SoongOutDir(), n)
		@@ -138,8 +201,10 @@ func runSoong(ctx Context, config Config) {
		}
		}

		integratedBp2Build := config.Environment().IsEnvTrue("INTEGRATED_BP2BUILD")

		// This is done unconditionally, but does not take a measurable amount of time
		bootstrapBlueprint(ctx, config)
		bootstrapBlueprint(ctx, config, integratedBp2Build)

		soongBuildEnv := config.Environment().Copy()
		soongBuildEnv.Set("TOP", os.Getenv("TOP"))
		@@ -164,13 +229,12 @@ func runSoong(ctx Context, config Config) {
		ctx.BeginTrace(metrics.RunSoong, "environment check")
		defer ctx.EndTrace()

		envFile := filepath.Join(config.SoongOutDir(), "soong.environment.used")
		getenv := func(k string) string {
		v, _ := soongBuildEnv.Get(k)
		return v
		}
		if stale, _ := shared.StaleEnvFile(envFile, getenv); stale {
		os.Remove(envFile)
		soongBuildEnvFile := filepath.Join(config.SoongOutDir(), usedEnvFile)
		checkEnvironmentFile(soongBuildEnv, soongBuildEnvFile)

		if integratedBp2Build {
		bp2buildEnvFile := filepath.Join(config.SoongOutDir(), usedEnvFile+".bp2build")
		checkEnvironmentFile(soongBuildEnv, bp2buildEnvFile)
		}
		}()