Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 56ebaf35 authored by Lukacs T. Berki's avatar Lukacs T. Berki
Browse files

Separate the files of bp2build and soong_build. 

The following files are forked:

- build-globs.ninja
- The glob list files
- .d files for the glob list files
- The output file (build.ninja or the bp2build marker file)

This makes bp2build and soong_build not overwrite each other's files
they need for proper incrementality.

Test: Presubmits.

Change-Id: I69d192cbd36ecd9677f46f3fa095dfce6f872227
parent 15c82a10

cmd/soong_build/main.go

+2 −10
Original line number Diff line number Diff line
@@ -472,10 +472,10 @@ func runBp2Build(configuration android.Config, extraNinjaDeps []string) { 
	// Generate out/soong/.bootstrap/build-globs.ninja with the actions to generate flattened globfiles

	// containing the globs seen during bp2build conversion

	if blueprintArgs.GlobFile != "" {

		bootstrap.WriteBuildGlobsNinjaFile(bootstrap.StageMain, bp2buildCtx.Context, blueprintArgs, configuration)

		bootstrap.WriteBuildGlobsNinjaFile(blueprintArgs.GlobListDir, bp2buildCtx.Context, blueprintArgs, configuration)

	}

	// Add the depfile on the expanded globs in out/soong/.primary/globs

	ninjaDeps = append(ninjaDeps, bootstrap.GlobFileListFiles(configuration)...)

	ninjaDeps = append(ninjaDeps, bootstrap.GlobFileListFiles(configuration, blueprintArgs.GlobListDir)...)



	// Run the code-generation phase to convert BazelTargetModules to BUILD files

	// and print conversion metrics to the user.
@@ -528,12 +528,4 @@ func runBp2Build(configuration android.Config, extraNinjaDeps []string) { 


	// Create an empty bp2build marker file.

	touch(shared.JoinPath(topDir, bp2buildMarker))



	// bp2build *always* writes a fake Ninja file containing just the nothing

	// phony target if it ever re-runs. This allows bp2build to exit early with

	// GENERATE_BAZEL_FILES=1 m nothing.

	//

	// If bp2build is invoked as part of an integrated mixed build, the fake

	// build.ninja file will be rewritten later into the real file anyway.

	writeFakeNinjaFile(extraNinjaDeps, codegenContext.Config().BuildDir())

}

tests/bootstrap_test.sh

+43 −11
Original line number Diff line number Diff line
@@ -144,7 +144,7 @@ EOF 
  run_soong

  local ninja_mtime1=$(stat -c "%y" out/soong/build.ninja)



  local glob_deps_file=out/soong/.primary/globs/0.d

  local glob_deps_file=out/soong/.bootstrap/globs/0.d



  if [ -e "$glob_deps_file" ]; then

    fail "Glob deps file unexpectedly written on first build"
@@ -526,18 +526,14 @@ function test_bp2build_smoke { 
  [[ -e out/soong/workspace ]] || fail "Bazel workspace not created"

}



function test_bp2build_generates_fake_ninja_file {

function test_bp2build_generates_marker_file {

  setup

  create_mock_bazel



  run_bp2build



  if [[ ! -f "./out/soong/build.ninja" ]]; then

    fail "./out/soong/build.ninja was not generated"

  fi



  if ! grep "build nothing: phony" "./out/soong/build.ninja"; then

    fail "missing phony nothing target in out/soong/build.ninja"

  if [[ ! -f "./out/soong/.bootstrap/bp2build_workspace_marker" ]]; then

    fail "Marker file was not generated"

  fi

}
@@ -577,10 +573,10 @@ function test_bp2build_null_build { 
  setup



  GENERATE_BAZEL_FILES=1 run_soong

  local mtime1=$(stat -c "%y" out/soong/build.ninja)

  local mtime1=$(stat -c "%y" out/soong/.bootstrap/bp2build_workspace_marker)



  GENERATE_BAZEL_FILES=1 run_soong

  local mtime2=$(stat -c "%y" out/soong/build.ninja)

  local mtime2=$(stat -c "%y" out/soong/.bootstrap/bp2build_workspace_marker)



  if [[ "$mtime1" != "$mtime2" ]]; then

    fail "Output Ninja file changed on null build"
@@ -712,6 +708,41 @@ EOF 
  grep -q "b/${GENERATED_BUILD_FILE_NAME}' exist" "$MOCK_TOP/errors" || fail "Error for b/${GENERATED_BUILD_FILE_NAME} not found"

}



function test_bp2build_back_and_forth_null_build {

  setup



  run_soong

  local output_mtime1=$(stat -c "%y" out/soong/build.ninja)



  GENERATE_BAZEL_FILES=1 run_soong

  local output_mtime2=$(stat -c "%y" out/soong/build.ninja)

  if [[ "$output_mtime1" != "$output_mtime2" ]]; then

    fail "Output Ninja file changed when switching to bp2build"

  fi



  local marker_mtime1=$(stat -c "%y" out/soong/.bootstrap/bp2build_workspace_marker)



  run_soong

  local output_mtime3=$(stat -c "%y" out/soong/build.ninja)

  local marker_mtime2=$(stat -c "%y" out/soong/.bootstrap/bp2build_workspace_marker)

  if [[ "$output_mtime1" != "$output_mtime3" ]]; then

    fail "Output Ninja file changed when switching to regular build from bp2build"

  fi

  if [[ "$marker_mtime1" != "$marker_mtime2" ]]; then

    fail "bp2build marker file changed when switching to regular build from bp2build"

  fi



  GENERATE_BAZEL_FILES=1 run_soong

  local output_mtime4=$(stat -c "%y" out/soong/build.ninja)

  local marker_mtime3=$(stat -c "%y" out/soong/.bootstrap/bp2build_workspace_marker)

  if [[ "$output_mtime1" != "$output_mtime4" ]]; then

    fail "Output Ninja file changed when switching back to bp2build"

  fi

  if [[ "$marker_mtime1" != "$marker_mtime3" ]]; then

    fail "bp2build marker file changed when switching back to bp2build"

  fi

}



test_smoke

test_null_build

test_null_build_after_docs
@@ -727,8 +758,9 @@ test_soong_build_rerun_iff_environment_changes 
test_dump_json_module_graph

test_write_to_source_tree

test_bp2build_smoke

test_bp2build_generates_fake_ninja_file

test_bp2build_generates_marker_file

test_bp2build_null_build

test_bp2build_back_and_forth_null_build

test_bp2build_add_android_bp

test_bp2build_add_to_glob

test_bp2build_bazel_workspace_structure

tests/bp2build_bazel_test.sh

+0 −54
Original line number Diff line number Diff line
@@ -48,60 +48,6 @@ EOF 


test_bp2build_null_build_with_globs



function test_soong_after_bp2build_regenerates_build_globs_ninja() {

  setup



  mkdir -p foo/bar

  cat > foo/bar/Android.bp <<'EOF'

filegroup {

    name: "bp2build-files",

    srcs: ["bp2build.*"],

    bazel_module: { bp2build_available: true },

}



filegroup {

    name: "soong-files",

    srcs: ["soong.*"],

}

EOF

  touch foo/bar/bp2build.txt foo/bar/soong.txt



  build_globs_file="out/soong/.bootstrap/build-globs.ninja"



  # Test: the build-globs file for bp2build should only contain the bp2build-files

  # glob, whereas the build-globs file for soong should contain both bp2build-files

  # and soong-files globs.



  run_bp2build

  local output_mtime1=$(stat -c "%y" "${build_globs_file}")



  if ! grep "\"foo/bar/bp2build.*\"" "${build_globs_file}"; then

    fail "bp2build filegroup globs not found in bp2build's globs file"

  fi



  if grep "\"foo/bar/soong.*\"" "${build_globs_file}"; then

    fail "soong filegroup globs unexpectedly found in bp2build's globs file"

  fi



  run_soong

  local output_mtime2=$(stat -c "%y" "${build_globs_file}")



  if [[ "$output_mtime1" == "$output_mtime2" ]]; then

    fail "Output build-globs.ninja file did not change"

  fi



  if ! grep "\"foo/bar/bp2build.*\"" "${build_globs_file}"; then

    fail "bp2build filegroup globs not found in bp2build's globs file"

  fi



  if ! grep "\"foo/bar/soong.*\"" "${build_globs_file}"; then

    fail "soong filegroup globs not found in bp2build's globs file"

  fi



}



test_soong_after_bp2build_regenerates_build_globs_ninja



function test_bp2build_generates_all_buildfiles {

  setup

  create_mock_bazel

ui/build/config.go

+8 −0
Original line number Diff line number Diff line
@@ -739,6 +739,14 @@ func (c *configImpl) SoongOutDir() string { 
	return filepath.Join(c.OutDir(), "soong")

}



func (c *configImpl) MainNinjaFile() string {

	return shared.JoinPath(c.SoongOutDir(), "build.ninja")

}



func (c *configImpl) Bp2BuildMarkerFile() string {

	return shared.JoinPath(c.SoongOutDir(), ".bootstrap/bp2build_workspace_marker")

}



func (c *configImpl) TempDir() string {

	return shared.TempDirForOutDir(c.SoongOutDir())

}

ui/build/soong.go

+54 −34
Original line number Diff line number Diff line
@@ -114,17 +114,20 @@ func writeEmptyGlobFile(ctx Context, path string) { 
	}

}



func bootstrapBlueprint(ctx Context, config Config, integratedBp2Build bool) {

func bootstrapBlueprint(ctx Context, config Config) {

	ctx.BeginTrace(metrics.RunSoong, "blueprint bootstrap")

	defer ctx.EndTrace()



	var args bootstrap.Args



	mainNinjaFile := shared.JoinPath(config.SoongOutDir(), "build.ninja")

	bootstrapGlobFile := shared.JoinPath(config.SoongOutDir(), ".bootstrap/build-globs.ninja")

	// .bootstrap/build.ninja "includes" .bootstrap/build-globs.ninja for incremental builds

	// generate an empty glob before running any rule in .bootstrap/build.ninja

	bp2buildGlobFile := shared.JoinPath(config.SoongOutDir(), ".bootstrap/build-globs.bp2build.ninja")



	// The glob .ninja files are subninja'd. However, they are generated during

	// the build itself so we write an empty file so that the subninja doesn't

	// fail on clean builds

	writeEmptyGlobFile(ctx, bootstrapGlobFile)

	writeEmptyGlobFile(ctx, bp2buildGlobFile)

	bootstrapDepFile := shared.JoinPath(config.SoongOutDir(), ".bootstrap/build.ninja.d")



	args.RunGoTests = !config.skipSoongTests
@@ -137,7 +140,7 @@ func bootstrapBlueprint(ctx Context, config Config, integratedBp2Build bool) { 
	// The primary builder (aka soong_build) will use bootstrapGlobFile as the globFile to generate build.ninja(.d)

	// Building soong_build does not require a glob file

	// Using "" instead of "<soong_build_glob>.ninja" will ensure that an unused glob file is not written to out/soong/.bootstrap during StagePrimary

	args.GlobFile = ""

	args.Subninjas = []string{bootstrapGlobFile, bp2buildGlobFile}

	args.GeneratingPrimaryBuilder = true

	args.EmptyNinjaFile = config.EmptyNinjaFile()
@@ -146,42 +149,46 @@ func bootstrapBlueprint(ctx Context, config Config, integratedBp2Build bool) { 
		args.DelvePath = shared.ResolveDelveBinary()

	}



	commonArgs := bootstrap.PrimaryBuilderExtraFlags(args, bootstrapGlobFile, mainNinjaFile)

	bp2BuildMarkerFile := shared.JoinPath(config.SoongOutDir(), ".bootstrap/bp2build_workspace_marker")

	commonArgs := bootstrap.PrimaryBuilderExtraFlags(args, config.MainNinjaFile())

	mainSoongBuildInputs := []string{"Android.bp"}



	if integratedBp2Build {

		mainSoongBuildInputs = append(mainSoongBuildInputs, bp2BuildMarkerFile)

	if config.bazelBuildMode() == mixedBuild {

		mainSoongBuildInputs = append(mainSoongBuildInputs, config.Bp2BuildMarkerFile())

	}



	soongBuildArgs := []string{

		"--globListDir", "globs",

		"--globFile", bootstrapGlobFile,

	}



	soongBuildArgs := make([]string, 0)

	soongBuildArgs = append(soongBuildArgs, commonArgs...)

	soongBuildArgs = append(soongBuildArgs, environmentArgs(config, "")...)

	soongBuildArgs = append(soongBuildArgs, "Android.bp")



	mainSoongBuildInvocation := bootstrap.PrimaryBuilderInvocation{

		Inputs:  mainSoongBuildInputs,

		Outputs: []string{mainNinjaFile},

		Outputs: []string{config.MainNinjaFile()},

		Args:    soongBuildArgs,

	}



	if integratedBp2Build {

		bp2buildArgs := []string{"--bp2build_marker", bp2BuildMarkerFile}

	bp2buildArgs := []string{

		"--bp2build_marker", config.Bp2BuildMarkerFile(),

		"--globListDir", "globs.bp2build",

		"--globFile", bp2buildGlobFile,

	}



	bp2buildArgs = append(bp2buildArgs, commonArgs...)

	bp2buildArgs = append(bp2buildArgs, environmentArgs(config, ".bp2build")...)

	bp2buildArgs = append(bp2buildArgs, "Android.bp")



	bp2buildInvocation := bootstrap.PrimaryBuilderInvocation{

		Inputs:  []string{"Android.bp"},

			Outputs: []string{bp2BuildMarkerFile},

		Outputs: []string{config.Bp2BuildMarkerFile()},

		Args:    bp2buildArgs,

	}

		args.PrimaryBuilderInvocations = []bootstrap.PrimaryBuilderInvocation{bp2buildInvocation}

		if config.bazelBuildMode() == mixedBuild {

			args.PrimaryBuilderInvocations = append(args.PrimaryBuilderInvocations, mainSoongBuildInvocation)

		}

	} else {

		args.PrimaryBuilderInvocations = []bootstrap.PrimaryBuilderInvocation{mainSoongBuildInvocation}

	args.PrimaryBuilderInvocations = []bootstrap.PrimaryBuilderInvocation{

		bp2buildInvocation,

		mainSoongBuildInvocation,

	}



	blueprintCtx := blueprint.NewContext()
@@ -231,7 +238,7 @@ func runSoong(ctx Context, config Config) { 
	integratedBp2Build := (buildMode == mixedBuild) || (buildMode == generateBuildFiles)



	// This is done unconditionally, but does not take a measurable amount of time

	bootstrapBlueprint(ctx, config, integratedBp2Build)

	bootstrapBlueprint(ctx, config)



	soongBuildEnv := config.Environment().Copy()

	soongBuildEnv.Set("TOP", os.Getenv("TOP"))
@@ -268,7 +275,7 @@ func runSoong(ctx Context, config Config) { 
	runMicrofactory(ctx, config, ".minibootstrap/bpglob", "github.com/google/blueprint/bootstrap/bpglob",

		map[string]string{"github.com/google/blueprint": "build/blueprint"})



	ninja := func(name, file string) {

	ninja := func(name, ninjaFile string, targets ...string) {

		ctx.BeginTrace(metrics.RunSoong, name)

		defer ctx.EndTrace()
@@ -276,8 +283,7 @@ func runSoong(ctx Context, config Config) { 
		nr := status.NewNinjaReader(ctx, ctx.Status.StartTool(), fifo)

		defer nr.Close()



		cmd := Command(ctx, config, "soong "+name,

			config.PrebuiltBuildTool("ninja"),

		ninjaArgs := []string{

			"-d", "keepdepfile",

			"-d", "stats",

			"-o", "usesphonyoutputs=yes",
@@ -287,7 +293,12 @@ func runSoong(ctx Context, config Config) { 
			"-w", "missingoutfile=err",

			"-j", strconv.Itoa(config.Parallel()),

			"--frontend_file", fifo,

			"-f", filepath.Join(config.SoongOutDir(), file))

			"-f", filepath.Join(config.SoongOutDir(), ninjaFile),

		}



		ninjaArgs = append(ninjaArgs, targets...)

		cmd := Command(ctx, config, "soong "+name,

			config.PrebuiltBuildTool("ninja"), ninjaArgs...)



		var ninjaEnv Environment
@@ -299,8 +310,17 @@ func runSoong(ctx Context, config Config) { 
		cmd.Sandbox = soongSandbox

		cmd.RunAndStreamOrFatal()

	}



	var target string



	if config.bazelBuildMode() == generateBuildFiles {

		target = config.Bp2BuildMarkerFile()

	} else {

		// This build generates <builddir>/build.ninja, which is used later by build/soong/ui/build/build.go#Build().

	ninja("bootstrap", ".bootstrap/build.ninja")

		target = config.MainNinjaFile()

	}



	ninja("bootstrap", ".bootstrap/build.ninja", target)



	var soongBuildMetrics *soong_metrics_proto.SoongBuildMetrics

	if shouldCollectBuildSoongMetrics(config) {