Separate the files of bp2build and soong_build. am: 56ebaf35b3 (8860d262) · Commits · e / os / android_build_soong

cmd/soong_build/main.go

+2 −10

Original line number	Diff line number	Diff line
		@@ -472,10 +472,10 @@ func runBp2Build(configuration android.Config, extraNinjaDeps []string) {
		// Generate out/soong/.bootstrap/build-globs.ninja with the actions to generate flattened globfiles
		// containing the globs seen during bp2build conversion
		if blueprintArgs.GlobFile != "" {
		bootstrap.WriteBuildGlobsNinjaFile(bootstrap.StageMain, bp2buildCtx.Context, blueprintArgs, configuration)
		bootstrap.WriteBuildGlobsNinjaFile(blueprintArgs.GlobListDir, bp2buildCtx.Context, blueprintArgs, configuration)
		}
		// Add the depfile on the expanded globs in out/soong/.primary/globs
		ninjaDeps = append(ninjaDeps, bootstrap.GlobFileListFiles(configuration)...)
		ninjaDeps = append(ninjaDeps, bootstrap.GlobFileListFiles(configuration, blueprintArgs.GlobListDir)...)

		// Run the code-generation phase to convert BazelTargetModules to BUILD files
		// and print conversion metrics to the user.
		@@ -528,12 +528,4 @@ func runBp2Build(configuration android.Config, extraNinjaDeps []string) {

		// Create an empty bp2build marker file.
		touch(shared.JoinPath(topDir, bp2buildMarker))

		// bp2build always writes a fake Ninja file containing just the nothing
		// phony target if it ever re-runs. This allows bp2build to exit early with
		// GENERATE_BAZEL_FILES=1 m nothing.
		//
		// If bp2build is invoked as part of an integrated mixed build, the fake
		// build.ninja file will be rewritten later into the real file anyway.
		writeFakeNinjaFile(extraNinjaDeps, codegenContext.Config().BuildDir())
		}

tests/bootstrap_test.sh

+43 −11

Original line number	Diff line number	Diff line
		@@ -144,7 +144,7 @@ EOF
		run_soong
		local ninja_mtime1=$(stat -c "%y" out/soong/build.ninja)

		local glob_deps_file=out/soong/.primary/globs/0.d
		local glob_deps_file=out/soong/.bootstrap/globs/0.d

		if [ -e "$glob_deps_file" ]; then
		fail "Glob deps file unexpectedly written on first build"
		@@ -526,18 +526,14 @@ function test_bp2build_smoke {
		[[ -e out/soong/workspace ]] \|\| fail "Bazel workspace not created"
		}

		function test_bp2build_generates_fake_ninja_file {
		function test_bp2build_generates_marker_file {
		setup
		create_mock_bazel

		run_bp2build

		if [[ ! -f "./out/soong/build.ninja" ]]; then
		fail "./out/soong/build.ninja was not generated"
		fi

		if ! grep "build nothing: phony" "./out/soong/build.ninja"; then
		fail "missing phony nothing target in out/soong/build.ninja"
		if [[ ! -f "./out/soong/.bootstrap/bp2build_workspace_marker" ]]; then
		fail "Marker file was not generated"
		fi
		}

		@@ -577,10 +573,10 @@ function test_bp2build_null_build {
		setup

		GENERATE_BAZEL_FILES=1 run_soong
		local mtime1=$(stat -c "%y" out/soong/build.ninja)
		local mtime1=$(stat -c "%y" out/soong/.bootstrap/bp2build_workspace_marker)

		GENERATE_BAZEL_FILES=1 run_soong
		local mtime2=$(stat -c "%y" out/soong/build.ninja)
		local mtime2=$(stat -c "%y" out/soong/.bootstrap/bp2build_workspace_marker)

		if [[ "$mtime1" != "$mtime2" ]]; then
		fail "Output Ninja file changed on null build"
		@@ -712,6 +708,41 @@ EOF
		grep -q "b/${GENERATED_BUILD_FILE_NAME}' exist" "$MOCK_TOP/errors" \|\| fail "Error for b/${GENERATED_BUILD_FILE_NAME} not found"
		}

		function test_bp2build_back_and_forth_null_build {
		setup

		run_soong
		local output_mtime1=$(stat -c "%y" out/soong/build.ninja)

		GENERATE_BAZEL_FILES=1 run_soong
		local output_mtime2=$(stat -c "%y" out/soong/build.ninja)
		if [[ "$output_mtime1" != "$output_mtime2" ]]; then
		fail "Output Ninja file changed when switching to bp2build"
		fi

		local marker_mtime1=$(stat -c "%y" out/soong/.bootstrap/bp2build_workspace_marker)

		run_soong
		local output_mtime3=$(stat -c "%y" out/soong/build.ninja)
		local marker_mtime2=$(stat -c "%y" out/soong/.bootstrap/bp2build_workspace_marker)
		if [[ "$output_mtime1" != "$output_mtime3" ]]; then
		fail "Output Ninja file changed when switching to regular build from bp2build"
		fi
		if [[ "$marker_mtime1" != "$marker_mtime2" ]]; then
		fail "bp2build marker file changed when switching to regular build from bp2build"
		fi

		GENERATE_BAZEL_FILES=1 run_soong
		local output_mtime4=$(stat -c "%y" out/soong/build.ninja)
		local marker_mtime3=$(stat -c "%y" out/soong/.bootstrap/bp2build_workspace_marker)
		if [[ "$output_mtime1" != "$output_mtime4" ]]; then
		fail "Output Ninja file changed when switching back to bp2build"
		fi
		if [[ "$marker_mtime1" != "$marker_mtime3" ]]; then
		fail "bp2build marker file changed when switching back to bp2build"
		fi
		}

		test_smoke
		test_null_build
		test_null_build_after_docs
		@@ -727,8 +758,9 @@ test_soong_build_rerun_iff_environment_changes
		test_dump_json_module_graph
		test_write_to_source_tree
		test_bp2build_smoke
		test_bp2build_generates_fake_ninja_file
		test_bp2build_generates_marker_file
		test_bp2build_null_build
		test_bp2build_back_and_forth_null_build
		test_bp2build_add_android_bp
		test_bp2build_add_to_glob
		test_bp2build_bazel_workspace_structure

tests/bp2build_bazel_test.sh

+0 −54

Original line number	Diff line number	Diff line
		@@ -48,60 +48,6 @@ EOF

		test_bp2build_null_build_with_globs

		function test_soong_after_bp2build_regenerates_build_globs_ninja() {
		setup

		mkdir -p foo/bar
		cat > foo/bar/Android.bp <<'EOF'
		filegroup {
		name: "bp2build-files",
		srcs: ["bp2build.*"],
		bazel_module: { bp2build_available: true },
		}

		filegroup {
		name: "soong-files",
		srcs: ["soong.*"],
		}
		EOF
		touch foo/bar/bp2build.txt foo/bar/soong.txt

		build_globs_file="out/soong/.bootstrap/build-globs.ninja"

		# Test: the build-globs file for bp2build should only contain the bp2build-files
		# glob, whereas the build-globs file for soong should contain both bp2build-files
		# and soong-files globs.

		run_bp2build
		local output_mtime1=$(stat -c "%y" "${build_globs_file}")

		if ! grep "\"foo/bar/bp2build.*\"" "${build_globs_file}"; then
		fail "bp2build filegroup globs not found in bp2build's globs file"
		fi

		if grep "\"foo/bar/soong.*\"" "${build_globs_file}"; then
		fail "soong filegroup globs unexpectedly found in bp2build's globs file"
		fi

		run_soong
		local output_mtime2=$(stat -c "%y" "${build_globs_file}")

		if [[ "$output_mtime1" == "$output_mtime2" ]]; then
		fail "Output build-globs.ninja file did not change"
		fi

		if ! grep "\"foo/bar/bp2build.*\"" "${build_globs_file}"; then
		fail "bp2build filegroup globs not found in bp2build's globs file"
		fi

		if ! grep "\"foo/bar/soong.*\"" "${build_globs_file}"; then
		fail "soong filegroup globs not found in bp2build's globs file"
		fi

		}

		test_soong_after_bp2build_regenerates_build_globs_ninja

		function test_bp2build_generates_all_buildfiles {
		setup
		create_mock_bazel

ui/build/config.go

+8 −0

Original line number	Diff line number	Diff line
		@@ -739,6 +739,14 @@ func (c *configImpl) SoongOutDir() string {
		return filepath.Join(c.OutDir(), "soong")
		}

		func (c *configImpl) MainNinjaFile() string {
		return shared.JoinPath(c.SoongOutDir(), "build.ninja")
		}

		func (c *configImpl) Bp2BuildMarkerFile() string {
		return shared.JoinPath(c.SoongOutDir(), ".bootstrap/bp2build_workspace_marker")
		}

		func (c *configImpl) TempDir() string {
		return shared.TempDirForOutDir(c.SoongOutDir())
		}

ui/build/soong.go

+54 −34

Original line number	Diff line number	Diff line
		@@ -114,17 +114,20 @@ func writeEmptyGlobFile(ctx Context, path string) {
		}
		}

		func bootstrapBlueprint(ctx Context, config Config, integratedBp2Build bool) {
		func bootstrapBlueprint(ctx Context, config Config) {
		ctx.BeginTrace(metrics.RunSoong, "blueprint bootstrap")
		defer ctx.EndTrace()

		var args bootstrap.Args

		mainNinjaFile := shared.JoinPath(config.SoongOutDir(), "build.ninja")
		bootstrapGlobFile := shared.JoinPath(config.SoongOutDir(), ".bootstrap/build-globs.ninja")
		// .bootstrap/build.ninja "includes" .bootstrap/build-globs.ninja for incremental builds
		// generate an empty glob before running any rule in .bootstrap/build.ninja
		bp2buildGlobFile := shared.JoinPath(config.SoongOutDir(), ".bootstrap/build-globs.bp2build.ninja")

		// The glob .ninja files are subninja'd. However, they are generated during
		// the build itself so we write an empty file so that the subninja doesn't
		// fail on clean builds
		writeEmptyGlobFile(ctx, bootstrapGlobFile)
		writeEmptyGlobFile(ctx, bp2buildGlobFile)
		bootstrapDepFile := shared.JoinPath(config.SoongOutDir(), ".bootstrap/build.ninja.d")

		args.RunGoTests = !config.skipSoongTests
		@@ -137,7 +140,7 @@ func bootstrapBlueprint(ctx Context, config Config, integratedBp2Build bool) {
		// The primary builder (aka soong_build) will use bootstrapGlobFile as the globFile to generate build.ninja(.d)
		// Building soong_build does not require a glob file
		// Using "" instead of "<soong_build_glob>.ninja" will ensure that an unused glob file is not written to out/soong/.bootstrap during StagePrimary
		args.GlobFile = ""
		args.Subninjas = []string{bootstrapGlobFile, bp2buildGlobFile}
		args.GeneratingPrimaryBuilder = true
		args.EmptyNinjaFile = config.EmptyNinjaFile()

		@@ -146,42 +149,46 @@ func bootstrapBlueprint(ctx Context, config Config, integratedBp2Build bool) {
		args.DelvePath = shared.ResolveDelveBinary()
		}

		commonArgs := bootstrap.PrimaryBuilderExtraFlags(args, bootstrapGlobFile, mainNinjaFile)
		bp2BuildMarkerFile := shared.JoinPath(config.SoongOutDir(), ".bootstrap/bp2build_workspace_marker")
		commonArgs := bootstrap.PrimaryBuilderExtraFlags(args, config.MainNinjaFile())
		mainSoongBuildInputs := []string{"Android.bp"}

		if integratedBp2Build {
		mainSoongBuildInputs = append(mainSoongBuildInputs, bp2BuildMarkerFile)
		if config.bazelBuildMode() == mixedBuild {
		mainSoongBuildInputs = append(mainSoongBuildInputs, config.Bp2BuildMarkerFile())
		}

		soongBuildArgs := []string{
		"--globListDir", "globs",
		"--globFile", bootstrapGlobFile,
		}

		soongBuildArgs := make([]string, 0)
		soongBuildArgs = append(soongBuildArgs, commonArgs...)
		soongBuildArgs = append(soongBuildArgs, environmentArgs(config, "")...)
		soongBuildArgs = append(soongBuildArgs, "Android.bp")

		mainSoongBuildInvocation := bootstrap.PrimaryBuilderInvocation{
		Inputs: mainSoongBuildInputs,
		Outputs: []string{mainNinjaFile},
		Outputs: []string{config.MainNinjaFile()},
		Args: soongBuildArgs,
		}

		if integratedBp2Build {
		bp2buildArgs := []string{"--bp2build_marker", bp2BuildMarkerFile}
		bp2buildArgs := []string{
		"--bp2build_marker", config.Bp2BuildMarkerFile(),
		"--globListDir", "globs.bp2build",
		"--globFile", bp2buildGlobFile,
		}

		bp2buildArgs = append(bp2buildArgs, commonArgs...)
		bp2buildArgs = append(bp2buildArgs, environmentArgs(config, ".bp2build")...)
		bp2buildArgs = append(bp2buildArgs, "Android.bp")

		bp2buildInvocation := bootstrap.PrimaryBuilderInvocation{
		Inputs: []string{"Android.bp"},
		Outputs: []string{bp2BuildMarkerFile},
		Outputs: []string{config.Bp2BuildMarkerFile()},
		Args: bp2buildArgs,
		}
		args.PrimaryBuilderInvocations = []bootstrap.PrimaryBuilderInvocation{bp2buildInvocation}
		if config.bazelBuildMode() == mixedBuild {
		args.PrimaryBuilderInvocations = append(args.PrimaryBuilderInvocations, mainSoongBuildInvocation)
		}
		} else {
		args.PrimaryBuilderInvocations = []bootstrap.PrimaryBuilderInvocation{mainSoongBuildInvocation}
		args.PrimaryBuilderInvocations = []bootstrap.PrimaryBuilderInvocation{
		bp2buildInvocation,
		mainSoongBuildInvocation,
		}

		blueprintCtx := blueprint.NewContext()
		@@ -231,7 +238,7 @@ func runSoong(ctx Context, config Config) {
		integratedBp2Build := (buildMode == mixedBuild) \|\| (buildMode == generateBuildFiles)

		// This is done unconditionally, but does not take a measurable amount of time
		bootstrapBlueprint(ctx, config, integratedBp2Build)
		bootstrapBlueprint(ctx, config)

		soongBuildEnv := config.Environment().Copy()
		soongBuildEnv.Set("TOP", os.Getenv("TOP"))
		@@ -268,7 +275,7 @@ func runSoong(ctx Context, config Config) {
		runMicrofactory(ctx, config, ".minibootstrap/bpglob", "github.com/google/blueprint/bootstrap/bpglob",
		map[string]string{"github.com/google/blueprint": "build/blueprint"})

		ninja := func(name, file string) {
		ninja := func(name, ninjaFile string, targets ...string) {
		ctx.BeginTrace(metrics.RunSoong, name)
		defer ctx.EndTrace()

		@@ -276,8 +283,7 @@ func runSoong(ctx Context, config Config) {
		nr := status.NewNinjaReader(ctx, ctx.Status.StartTool(), fifo)
		defer nr.Close()

		cmd := Command(ctx, config, "soong "+name,
		config.PrebuiltBuildTool("ninja"),
		ninjaArgs := []string{
		"-d", "keepdepfile",
		"-d", "stats",
		"-o", "usesphonyoutputs=yes",
		@@ -287,7 +293,12 @@ func runSoong(ctx Context, config Config) {
		"-w", "missingoutfile=err",
		"-j", strconv.Itoa(config.Parallel()),
		"--frontend_file", fifo,
		"-f", filepath.Join(config.SoongOutDir(), file))
		"-f", filepath.Join(config.SoongOutDir(), ninjaFile),
		}

		ninjaArgs = append(ninjaArgs, targets...)
		cmd := Command(ctx, config, "soong "+name,
		config.PrebuiltBuildTool("ninja"), ninjaArgs...)

		var ninjaEnv Environment

		@@ -299,8 +310,17 @@ func runSoong(ctx Context, config Config) {
		cmd.Sandbox = soongSandbox
		cmd.RunAndStreamOrFatal()
		}

		var target string

		if config.bazelBuildMode() == generateBuildFiles {
		target = config.Bp2BuildMarkerFile()
		} else {
		// This build generates <builddir>/build.ninja, which is used later by build/soong/ui/build/build.go#Build().
		ninja("bootstrap", ".bootstrap/build.ninja")
		target = config.MainNinjaFile()
		}

		ninja("bootstrap", ".bootstrap/build.ninja", target)

		var soongBuildMetrics *soong_metrics_proto.SoongBuildMetrics
		if shouldCollectBuildSoongMetrics(config) {