Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit e571dc3b authored by Lukacs T. Berki's avatar Lukacs T. Berki
Browse files

Print the JSON module graph correctly. 

Before, we piggybacked on the implementation of regular soong_build and
wrote a fake build.ninja file to satisfy Ninja.

Now, instead, the JSON module graph is a a separate action in the Ninja
output file. This has the pleasant side effect that one can flip back
and forth between generating the JSON file and regular Soong without
loss of incrementality.

Side cleanup: write .d files in a slightly cleaner way.

Test: Presubmits.
Change-Id: Ia853383567b9dd31c53f3bdf56cfc8d517b498ec
parent 5602d330

cmd/soong_build/main.go

+29 −55
Original line number Diff line number Diff line
@@ -25,6 +25,7 @@ import ( 


	"android/soong/bp2build"

	"android/soong/shared"



	"github.com/google/blueprint/bootstrap"

	"github.com/google/blueprint/deptools"

	"github.com/google/blueprint/pathtools"
@@ -43,6 +44,7 @@ var ( 
	delveListen string

	delvePath   string



	moduleGraphFile   string

	docFile           string

	bazelQueryViewDir string

	bp2buildMarker    string
@@ -62,6 +64,7 @@ func init() { 
	flag.StringVar(&delvePath, "delve_path", "", "Path to Delve. Only used if --delve_listen is set")



	// Flags representing various modes soong_build can run in

	flag.StringVar(&moduleGraphFile, "module_graph_file", "", "JSON module graph file to output")

	flag.StringVar(&docFile, "soong_docs", "", "build documentation file to output")

	flag.StringVar(&bazelQueryViewDir, "bazel_queryview_dir", "", "path to the bazel queryview directory relative to --top")

	flag.StringVar(&bp2buildMarker, "bp2build_marker", "", "If set, run bp2build, touch the specified marker file then exit")
@@ -71,7 +74,6 @@ func init() { 
	flag.StringVar(&globListDir, "globListDir", "", "the directory containing the glob list files")

	flag.StringVar(&cmdlineArgs.BuildDir, "b", ".", "the build output directory")

	flag.StringVar(&cmdlineArgs.NinjaBuildDir, "n", "", "the ninja builddir directory")

	flag.StringVar(&cmdlineArgs.DepFile, "d", "", "the dependency file to output")

	flag.StringVar(&cmdlineArgs.Cpuprofile, "cpuprofile", "", "write cpu profile to file")

	flag.StringVar(&cmdlineArgs.TraceFile, "trace", "", "write trace to file")

	flag.StringVar(&cmdlineArgs.Memprofile, "memprofile", "", "write memory profile to file")
@@ -149,11 +151,7 @@ func runMixedModeBuild(configuration android.Config, firstCtx *android.Context, 
	globListFiles := writeBuildGlobsNinjaFile(secondCtx.SrcDir(), configuration.SoongOutDir(), secondCtx.Globs, configuration)

	ninjaDeps = append(ninjaDeps, globListFiles...)



	err = deptools.WriteDepFile(shared.JoinPath(topDir, secondArgs.DepFile), secondArgs.OutFile, ninjaDeps)

	if err != nil {

		fmt.Fprintf(os.Stderr, "Error writing depfile '%s': %s\n", secondArgs.DepFile, err)

		os.Exit(1)

	}

	writeDepFile(secondArgs.OutFile, ninjaDeps)

}



// Run the code-generation phase to convert BazelTargetModules to BUILD files.
@@ -185,8 +183,8 @@ func writeMetrics(configuration android.Config) { 
	}

}



func writeJsonModuleGraph(configuration android.Config, ctx *android.Context, path string, extraNinjaDeps []string) {

	f, err := os.Create(path)

func writeJsonModuleGraph(ctx *android.Context, path string) {

	f, err := os.Create(shared.JoinPath(topDir, path))

	if err != nil {

		fmt.Fprintf(os.Stderr, "%s", err)

		os.Exit(1)
@@ -194,7 +192,6 @@ func writeJsonModuleGraph(configuration android.Config, ctx *android.Context, pa 


	defer f.Close()

	ctx.Context.PrintJSONGraph(f)

	writeFakeNinjaFile(extraNinjaDeps, configuration.SoongOutDir())

}



func writeBuildGlobsNinjaFile(srcDir, buildDir string, globs func() pathtools.MultipleGlobResults, config interface{}) []string {
@@ -208,6 +205,15 @@ func writeBuildGlobsNinjaFile(srcDir, buildDir string, globs func() pathtools.Mu 
	return bootstrap.GlobFileListFiles(globDir)

}



func writeDepFile(outputFile string, ninjaDeps []string) {

	depFile := shared.JoinPath(topDir, outputFile+".d")

	err := deptools.WriteDepFile(depFile, outputFile, ninjaDeps)

	if err != nil {

		fmt.Fprintf(os.Stderr, "Error writing depfile '%s': %s\n", depFile, err)

		os.Exit(1)

	}

}



// doChosenActivity runs Soong for a specific activity, like bp2build, queryview

// or the actual Soong build for the build.ninja file. Returns the top level

// output file of the specific activity.
@@ -215,10 +221,9 @@ func doChosenActivity(configuration android.Config, extraNinjaDeps []string) str 
	bazelConversionRequested := bp2buildMarker != ""

	mixedModeBuild := configuration.BazelContext.BazelEnabled()

	generateQueryView := bazelQueryViewDir != ""

	jsonModuleFile := configuration.Getenv("SOONG_DUMP_JSON_MODULE_GRAPH")



	blueprintArgs := cmdlineArgs

	prepareBuildActions := !generateQueryView && jsonModuleFile == ""

	prepareBuildActions := !generateQueryView && moduleGraphFile == ""

	if bazelConversionRequested {

		// Run the alternate pipeline of bp2build mutators and singleton to convert

		// Blueprint to BUILD files before everything else.
@@ -236,22 +241,19 @@ func doChosenActivity(configuration android.Config, extraNinjaDeps []string) str 
		globListFiles := writeBuildGlobsNinjaFile(ctx.SrcDir(), configuration.SoongOutDir(), ctx.Globs, configuration)

		ninjaDeps = append(ninjaDeps, globListFiles...)



		err := deptools.WriteDepFile(shared.JoinPath(topDir, blueprintArgs.DepFile), blueprintArgs.OutFile, ninjaDeps)

		if err != nil {

			fmt.Fprintf(os.Stderr, "Error writing depfile '%s': %s\n", blueprintArgs.DepFile, err)

			os.Exit(1)

		}

	}



		// Convert the Soong module graph into Bazel BUILD files.

		if generateQueryView {

			runQueryView(configuration, ctx)

			return cmdlineArgs.OutFile // TODO: This is a lie

		} else if moduleGraphFile != "" {

			writeJsonModuleGraph(ctx, moduleGraphFile)

			writeDepFile(moduleGraphFile, ninjaDeps)

			return moduleGraphFile

		} else {

			// The actual output (build.ninja) was written in the RunBlueprint() call

			// above

			writeDepFile(cmdlineArgs.OutFile, ninjaDeps)

		}



	if jsonModuleFile != "" {

		writeJsonModuleGraph(configuration, ctx, jsonModuleFile, extraNinjaDeps)

		return cmdlineArgs.OutFile // TODO: This is a lie

	}



	writeMetrics(configuration)
@@ -348,29 +350,6 @@ func writeUsedEnvironmentFile(configuration android.Config, finalOutputFile stri 
	touch(shared.JoinPath(topDir, finalOutputFile))

}



// Workarounds to support running bp2build in a clean AOSP checkout with no

// prior builds, and exiting early as soon as the BUILD files get generated,

// therefore not creating build.ninja files that soong_ui and callers of

// soong_build expects.

//

// These files are: build.ninja and build.ninja.d. Since Kati hasn't been

// ran as well, and `nothing` is defined in a .mk file, there isn't a ninja

// target called `nothing`, so we manually create it here.

func writeFakeNinjaFile(extraNinjaDeps []string, soongOutDir string) {

	extraNinjaDepsString := strings.Join(extraNinjaDeps, " \\\n ")



	ninjaFileName := "build.ninja"

	ninjaFile := shared.JoinPath(topDir, soongOutDir, ninjaFileName)

	ninjaFileD := shared.JoinPath(topDir, soongOutDir, ninjaFileName+".d")

	// A workaround to create the 'nothing' ninja target so `m nothing` works,

	// since bp2build runs without Kati, and the 'nothing' target is declared in

	// a Makefile.

	ioutil.WriteFile(ninjaFile, []byte("build nothing: phony\n  phony_output = true\n"), 0666)

	ioutil.WriteFile(ninjaFileD,

		[]byte(fmt.Sprintf("%s: \\\n %s\n", ninjaFile, extraNinjaDepsString)),

		0666)

}



func touch(path string) {

	f, err := os.OpenFile(path, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0666)

	if err != nil {
@@ -550,12 +529,7 @@ func runBp2Build(configuration android.Config, extraNinjaDeps []string) { 
	ninjaDeps = append(ninjaDeps, codegenContext.AdditionalNinjaDeps()...)

	ninjaDeps = append(ninjaDeps, symlinkForestDeps...)



	depFile := bp2buildMarker + ".d"

	err = deptools.WriteDepFile(shared.JoinPath(topDir, depFile), bp2buildMarker, ninjaDeps)

	if err != nil {

		fmt.Fprintf(os.Stderr, "Cannot write depfile '%s': %s\n", depFile, err)

		os.Exit(1)

	}

	writeDepFile(bp2buildMarker, ninjaDeps)



	// Create an empty bp2build marker file.

	touch(shared.JoinPath(topDir, bp2buildMarker))

tests/bootstrap_test.sh

+27 −2
Original line number Diff line number Diff line
@@ -607,12 +607,36 @@ EOF 


function test_dump_json_module_graph() {

  setup

  SOONG_DUMP_JSON_MODULE_GRAPH="$MOCK_TOP/modules.json" run_soong

  if [[ ! -r "$MOCK_TOP/modules.json" ]]; then

  GENERATE_JSON_MODULE_GRAPH=1 run_soong

  if [[ ! -r "out/soong//module-graph.json" ]]; then

    fail "JSON file was not created"

  fi

}



function test_json_module_graph_back_and_forth_null_build() {

  setup



  run_soong

  local ninja_mtime1=$(stat -c "%y" out/soong/build.ninja)



  GENERATE_JSON_MODULE_GRAPH=1 run_soong

  local json_mtime1=$(stat -c "%y" out/soong/module-graph.json)



  run_soong

  local ninja_mtime2=$(stat -c "%y" out/soong/build.ninja)

  if [[ "$ninja_mtime1" != "$ninja_mtime2" ]]; then

    fail "Output Ninja file changed after writing JSON module graph"

  fi



  GENERATE_JSON_MODULE_GRAPH=1 run_soong

  local json_mtime2=$(stat -c "%y" out/soong/module-graph.json)

  if [[ "$json_mtime1" != "$json_mtime2" ]]; then

    fail "JSON module graph file changed after writing Ninja file"

  fi



}





function test_bp2build_bazel_workspace_structure {

  setup
@@ -757,6 +781,7 @@ test_add_file_to_soong_build 
test_glob_during_bootstrapping

test_soong_build_rerun_iff_environment_changes

test_dump_json_module_graph

test_json_module_graph_back_and_forth_null_build

test_write_to_source_tree

test_bp2build_smoke

test_bp2build_generates_marker_file

ui/build/config.go

+5 −1
Original line number Diff line number Diff line
@@ -747,6 +747,10 @@ func (c *configImpl) Bp2BuildMarkerFile() string { 
	return shared.JoinPath(c.SoongOutDir(), ".bootstrap/bp2build_workspace_marker")

}



func (c *configImpl) ModuleGraphFile() string {

	return shared.JoinPath(c.SoongOutDir(), "module-graph.json")

}



func (c *configImpl) TempDir() string {

	return shared.TempDirForOutDir(c.SoongOutDir())

}
@@ -919,7 +923,7 @@ func (c *configImpl) bazelBuildMode() bazelBuildMode { 
		return mixedBuild

	} else if c.Environment().IsEnvTrue("GENERATE_BAZEL_FILES") {

		return generateBuildFiles

	} else if v, ok := c.Environment().Get("SOONG_DUMP_JSON_MODULE_GRAPH"); ok && v != "" {

	} else if c.Environment().IsEnvTrue("GENERATE_JSON_MODULE_GRAPH") {

		return generateJsonModuleGraph

	} else {

		return noBazel

ui/build/soong.go

+21 −0
Original line number Diff line number Diff line
@@ -117,6 +117,7 @@ func bootstrapBlueprint(ctx Context, config Config) { 


	bootstrapGlobFile := shared.JoinPath(config.SoongOutDir(), ".bootstrap/build-globs.ninja")

	bp2buildGlobFile := shared.JoinPath(config.SoongOutDir(), ".bootstrap/build-globs.bp2build.ninja")

	moduleGraphGlobFile := shared.JoinPath(config.SoongOutDir(), ".bootstrap/build-globs.modulegraph.ninja")



	// The glob .ninja files are subninja'd. However, they are generated during

	// the build itself so we write an empty file so that the subninja doesn't
@@ -181,9 +182,27 @@ func bootstrapBlueprint(ctx Context, config Config) { 
		Outputs: []string{config.Bp2BuildMarkerFile()},

		Args:    bp2buildArgs,

	}



	moduleGraphArgs := []string{

		"--module_graph_file", config.ModuleGraphFile(),

		"--globListDir", "globs.modulegraph",

		"--globFile", moduleGraphGlobFile,

	}



	moduleGraphArgs = append(moduleGraphArgs, commonArgs...)

	moduleGraphArgs = append(moduleGraphArgs, environmentArgs(config, ".modulegraph")...)

	moduleGraphArgs = append(moduleGraphArgs, "Android.bp")



	moduleGraphInvocation := bootstrap.PrimaryBuilderInvocation{

		Inputs:  []string{"Android.bp"},

		Outputs: []string{config.ModuleGraphFile()},

		Args:    moduleGraphArgs,

	}



	args.PrimaryBuilderInvocations = []bootstrap.PrimaryBuilderInvocation{

		bp2buildInvocation,

		mainSoongBuildInvocation,

		moduleGraphInvocation,

	}



	blueprintCtx := blueprint.NewContext()
@@ -307,6 +326,8 @@ func runSoong(ctx Context, config Config) { 


	if config.bazelBuildMode() == generateBuildFiles {

		target = config.Bp2BuildMarkerFile()

	} else if config.bazelBuildMode() == generateJsonModuleGraph {

		target = config.ModuleGraphFile()

	} else {

		// This build generates <builddir>/build.ninja, which is used later by build/soong/ui/build/build.go#Build().

		target = config.MainNinjaFile()