Merge "Print the JSON module graph correctly." (b8e5cb05) · Commits · e / os / android_build_soong

cmd/soong_build/main.go

+29 −55

Original line number	Diff line number	Diff line
		@@ -25,6 +25,7 @@ import (

		"android/soong/bp2build"
		"android/soong/shared"

		"github.com/google/blueprint/bootstrap"
		"github.com/google/blueprint/deptools"
		"github.com/google/blueprint/pathtools"
		@@ -43,6 +44,7 @@ var (
		delveListen string
		delvePath string

		moduleGraphFile string
		docFile string
		bazelQueryViewDir string
		bp2buildMarker string
		@@ -62,6 +64,7 @@ func init() {
		flag.StringVar(&delvePath, "delve_path", "", "Path to Delve. Only used if --delve_listen is set")

		// Flags representing various modes soong_build can run in
		flag.StringVar(&moduleGraphFile, "module_graph_file", "", "JSON module graph file to output")
		flag.StringVar(&docFile, "soong_docs", "", "build documentation file to output")
		flag.StringVar(&bazelQueryViewDir, "bazel_queryview_dir", "", "path to the bazel queryview directory relative to --top")
		flag.StringVar(&bp2buildMarker, "bp2build_marker", "", "If set, run bp2build, touch the specified marker file then exit")
		@@ -71,7 +74,6 @@ func init() {
		flag.StringVar(&globListDir, "globListDir", "", "the directory containing the glob list files")
		flag.StringVar(&cmdlineArgs.BuildDir, "b", ".", "the build output directory")
		flag.StringVar(&cmdlineArgs.NinjaBuildDir, "n", "", "the ninja builddir directory")
		flag.StringVar(&cmdlineArgs.DepFile, "d", "", "the dependency file to output")
		flag.StringVar(&cmdlineArgs.Cpuprofile, "cpuprofile", "", "write cpu profile to file")
		flag.StringVar(&cmdlineArgs.TraceFile, "trace", "", "write trace to file")
		flag.StringVar(&cmdlineArgs.Memprofile, "memprofile", "", "write memory profile to file")
		@@ -149,11 +151,7 @@ func runMixedModeBuild(configuration android.Config, firstCtx *android.Context,
		globListFiles := writeBuildGlobsNinjaFile(secondCtx.SrcDir(), configuration.SoongOutDir(), secondCtx.Globs, configuration)
		ninjaDeps = append(ninjaDeps, globListFiles...)

		err = deptools.WriteDepFile(shared.JoinPath(topDir, secondArgs.DepFile), secondArgs.OutFile, ninjaDeps)
		if err != nil {
		fmt.Fprintf(os.Stderr, "Error writing depfile '%s': %s\n", secondArgs.DepFile, err)
		os.Exit(1)
		}
		writeDepFile(secondArgs.OutFile, ninjaDeps)
		}

		// Run the code-generation phase to convert BazelTargetModules to BUILD files.
		@@ -185,8 +183,8 @@ func writeMetrics(configuration android.Config) {
		}
		}

		func writeJsonModuleGraph(configuration android.Config, ctx *android.Context, path string, extraNinjaDeps []string) {
		f, err := os.Create(path)
		func writeJsonModuleGraph(ctx *android.Context, path string) {
		f, err := os.Create(shared.JoinPath(topDir, path))
		if err != nil {
		fmt.Fprintf(os.Stderr, "%s", err)
		os.Exit(1)
		@@ -194,7 +192,6 @@ func writeJsonModuleGraph(configuration android.Config, ctx *android.Context, pa

		defer f.Close()
		ctx.Context.PrintJSONGraph(f)
		writeFakeNinjaFile(extraNinjaDeps, configuration.SoongOutDir())
		}

		func writeBuildGlobsNinjaFile(srcDir, buildDir string, globs func() pathtools.MultipleGlobResults, config interface{}) []string {
		@@ -208,6 +205,15 @@ func writeBuildGlobsNinjaFile(srcDir, buildDir string, globs func() pathtools.Mu
		return bootstrap.GlobFileListFiles(globDir)
		}

		func writeDepFile(outputFile string, ninjaDeps []string) {
		depFile := shared.JoinPath(topDir, outputFile+".d")
		err := deptools.WriteDepFile(depFile, outputFile, ninjaDeps)
		if err != nil {
		fmt.Fprintf(os.Stderr, "Error writing depfile '%s': %s\n", depFile, err)
		os.Exit(1)
		}
		}

		// doChosenActivity runs Soong for a specific activity, like bp2build, queryview
		// or the actual Soong build for the build.ninja file. Returns the top level
		// output file of the specific activity.
		@@ -215,10 +221,9 @@ func doChosenActivity(configuration android.Config, extraNinjaDeps []string) str
		bazelConversionRequested := bp2buildMarker != ""
		mixedModeBuild := configuration.BazelContext.BazelEnabled()
		generateQueryView := bazelQueryViewDir != ""
		jsonModuleFile := configuration.Getenv("SOONG_DUMP_JSON_MODULE_GRAPH")

		blueprintArgs := cmdlineArgs
		prepareBuildActions := !generateQueryView && jsonModuleFile == ""
		prepareBuildActions := !generateQueryView && moduleGraphFile == ""
		if bazelConversionRequested {
		// Run the alternate pipeline of bp2build mutators and singleton to convert
		// Blueprint to BUILD files before everything else.
		@@ -236,22 +241,19 @@ func doChosenActivity(configuration android.Config, extraNinjaDeps []string) str
		globListFiles := writeBuildGlobsNinjaFile(ctx.SrcDir(), configuration.SoongOutDir(), ctx.Globs, configuration)
		ninjaDeps = append(ninjaDeps, globListFiles...)

		err := deptools.WriteDepFile(shared.JoinPath(topDir, blueprintArgs.DepFile), blueprintArgs.OutFile, ninjaDeps)
		if err != nil {
		fmt.Fprintf(os.Stderr, "Error writing depfile '%s': %s\n", blueprintArgs.DepFile, err)
		os.Exit(1)
		}
		}

		// Convert the Soong module graph into Bazel BUILD files.
		if generateQueryView {
		runQueryView(configuration, ctx)
		return cmdlineArgs.OutFile // TODO: This is a lie
		} else if moduleGraphFile != "" {
		writeJsonModuleGraph(ctx, moduleGraphFile)
		writeDepFile(moduleGraphFile, ninjaDeps)
		return moduleGraphFile
		} else {
		// The actual output (build.ninja) was written in the RunBlueprint() call
		// above
		writeDepFile(cmdlineArgs.OutFile, ninjaDeps)
		}

		if jsonModuleFile != "" {
		writeJsonModuleGraph(configuration, ctx, jsonModuleFile, extraNinjaDeps)
		return cmdlineArgs.OutFile // TODO: This is a lie
		}

		writeMetrics(configuration)
		@@ -348,29 +350,6 @@ func writeUsedEnvironmentFile(configuration android.Config, finalOutputFile stri
		touch(shared.JoinPath(topDir, finalOutputFile))
		}

		// Workarounds to support running bp2build in a clean AOSP checkout with no
		// prior builds, and exiting early as soon as the BUILD files get generated,
		// therefore not creating build.ninja files that soong_ui and callers of
		// soong_build expects.
		//
		// These files are: build.ninja and build.ninja.d. Since Kati hasn't been
		// ran as well, and `nothing` is defined in a .mk file, there isn't a ninja
		// target called `nothing`, so we manually create it here.
		func writeFakeNinjaFile(extraNinjaDeps []string, soongOutDir string) {
		extraNinjaDepsString := strings.Join(extraNinjaDeps, " \\\n ")

		ninjaFileName := "build.ninja"
		ninjaFile := shared.JoinPath(topDir, soongOutDir, ninjaFileName)
		ninjaFileD := shared.JoinPath(topDir, soongOutDir, ninjaFileName+".d")
		// A workaround to create the 'nothing' ninja target so `m nothing` works,
		// since bp2build runs without Kati, and the 'nothing' target is declared in
		// a Makefile.
		ioutil.WriteFile(ninjaFile, []byte("build nothing: phony\n phony_output = true\n"), 0666)
		ioutil.WriteFile(ninjaFileD,
		[]byte(fmt.Sprintf("%s: \\\n %s\n", ninjaFile, extraNinjaDepsString)),
		0666)
		}

		func touch(path string) {
		f, err := os.OpenFile(path, os.O_APPEND\|os.O_CREATE\|os.O_WRONLY, 0666)
		if err != nil {
		@@ -550,12 +529,7 @@ func runBp2Build(configuration android.Config, extraNinjaDeps []string) {
		ninjaDeps = append(ninjaDeps, codegenContext.AdditionalNinjaDeps()...)
		ninjaDeps = append(ninjaDeps, symlinkForestDeps...)

		depFile := bp2buildMarker + ".d"
		err = deptools.WriteDepFile(shared.JoinPath(topDir, depFile), bp2buildMarker, ninjaDeps)
		if err != nil {
		fmt.Fprintf(os.Stderr, "Cannot write depfile '%s': %s\n", depFile, err)
		os.Exit(1)
		}
		writeDepFile(bp2buildMarker, ninjaDeps)

		// Create an empty bp2build marker file.
		touch(shared.JoinPath(topDir, bp2buildMarker))

tests/bootstrap_test.sh

+27 −2

Original line number	Diff line number	Diff line
		@@ -607,12 +607,36 @@ EOF

		function test_dump_json_module_graph() {
		setup
		SOONG_DUMP_JSON_MODULE_GRAPH="$MOCK_TOP/modules.json" run_soong
		if [[ ! -r "$MOCK_TOP/modules.json" ]]; then
		GENERATE_JSON_MODULE_GRAPH=1 run_soong
		if [[ ! -r "out/soong//module-graph.json" ]]; then
		fail "JSON file was not created"
		fi
		}

		function test_json_module_graph_back_and_forth_null_build() {
		setup

		run_soong
		local ninja_mtime1=$(stat -c "%y" out/soong/build.ninja)

		GENERATE_JSON_MODULE_GRAPH=1 run_soong
		local json_mtime1=$(stat -c "%y" out/soong/module-graph.json)

		run_soong
		local ninja_mtime2=$(stat -c "%y" out/soong/build.ninja)
		if [[ "$ninja_mtime1" != "$ninja_mtime2" ]]; then
		fail "Output Ninja file changed after writing JSON module graph"
		fi

		GENERATE_JSON_MODULE_GRAPH=1 run_soong
		local json_mtime2=$(stat -c "%y" out/soong/module-graph.json)
		if [[ "$json_mtime1" != "$json_mtime2" ]]; then
		fail "JSON module graph file changed after writing Ninja file"
		fi

		}


		function test_bp2build_bazel_workspace_structure {
		setup

		@@ -757,6 +781,7 @@ test_add_file_to_soong_build
		test_glob_during_bootstrapping
		test_soong_build_rerun_iff_environment_changes
		test_dump_json_module_graph
		test_json_module_graph_back_and_forth_null_build
		test_write_to_source_tree
		test_bp2build_smoke
		test_bp2build_generates_marker_file

ui/build/config.go

+5 −1

Original line number	Diff line number	Diff line
		@@ -747,6 +747,10 @@ func (c *configImpl) Bp2BuildMarkerFile() string {
		return shared.JoinPath(c.SoongOutDir(), ".bootstrap/bp2build_workspace_marker")
		}

		func (c *configImpl) ModuleGraphFile() string {
		return shared.JoinPath(c.SoongOutDir(), "module-graph.json")
		}

		func (c *configImpl) TempDir() string {
		return shared.TempDirForOutDir(c.SoongOutDir())
		}
		@@ -919,7 +923,7 @@ func (c *configImpl) bazelBuildMode() bazelBuildMode {
		return mixedBuild
		} else if c.Environment().IsEnvTrue("GENERATE_BAZEL_FILES") {
		return generateBuildFiles
		} else if v, ok := c.Environment().Get("SOONG_DUMP_JSON_MODULE_GRAPH"); ok && v != "" {
		} else if c.Environment().IsEnvTrue("GENERATE_JSON_MODULE_GRAPH") {
		return generateJsonModuleGraph
		} else {
		return noBazel

ui/build/soong.go

+21 −0

Original line number	Diff line number	Diff line
		@@ -117,6 +117,7 @@ func bootstrapBlueprint(ctx Context, config Config) {

		bootstrapGlobFile := shared.JoinPath(config.SoongOutDir(), ".bootstrap/build-globs.ninja")
		bp2buildGlobFile := shared.JoinPath(config.SoongOutDir(), ".bootstrap/build-globs.bp2build.ninja")
		moduleGraphGlobFile := shared.JoinPath(config.SoongOutDir(), ".bootstrap/build-globs.modulegraph.ninja")

		// The glob .ninja files are subninja'd. However, they are generated during
		// the build itself so we write an empty file so that the subninja doesn't
		@@ -181,9 +182,27 @@ func bootstrapBlueprint(ctx Context, config Config) {
		Outputs: []string{config.Bp2BuildMarkerFile()},
		Args: bp2buildArgs,
		}

		moduleGraphArgs := []string{
		"--module_graph_file", config.ModuleGraphFile(),
		"--globListDir", "globs.modulegraph",
		"--globFile", moduleGraphGlobFile,
		}

		moduleGraphArgs = append(moduleGraphArgs, commonArgs...)
		moduleGraphArgs = append(moduleGraphArgs, environmentArgs(config, ".modulegraph")...)
		moduleGraphArgs = append(moduleGraphArgs, "Android.bp")

		moduleGraphInvocation := bootstrap.PrimaryBuilderInvocation{
		Inputs: []string{"Android.bp"},
		Outputs: []string{config.ModuleGraphFile()},
		Args: moduleGraphArgs,
		}

		args.PrimaryBuilderInvocations = []bootstrap.PrimaryBuilderInvocation{
		bp2buildInvocation,
		mainSoongBuildInvocation,
		moduleGraphInvocation,
		}

		blueprintCtx := blueprint.NewContext()
		@@ -307,6 +326,8 @@ func runSoong(ctx Context, config Config) {

		if config.bazelBuildMode() == generateBuildFiles {
		target = config.Bp2BuildMarkerFile()
		} else if config.bazelBuildMode() == generateJsonModuleGraph {
		target = config.ModuleGraphFile()
		} else {
		// This build generates <builddir>/build.ninja, which is used later by build/soong/ui/build/build.go#Build().
		target = config.MainNinjaFile()