Merge "Create a synthetic Bazel workspace." (3784448c) · Commits · e / os / android_build_soong

bp2build/Android.bp

+1 −0

Original line number	Diff line number	Diff line
		@@ -14,6 +14,7 @@ bootstrap_go_package {
		"constants.go",
		"conversion.go",
		"metrics.go",
		"symlink_forest.go",
		],
		deps: [
		"soong-android",

bp2build/conversion_test.go

+2 −2

Original line number	Diff line number	Diff line
		@@ -19,14 +19,14 @@ import (
		"testing"
		)

		type filepath struct {
		type bazelFilepath struct {
		dir string
		basename string
		}

		func TestCreateBazelFiles_QueryView_AddsTopLevelFiles(t *testing.T) {
		files := CreateBazelFiles(map[string]RuleShim{}, map[string]BazelTargets{}, QueryView)
		expectedFilePaths := []filepath{
		expectedFilePaths := []bazelFilepath{
		{
		dir: "",
		basename: "BUILD",

bp2build/symlink_forest.go

0 → 100644

+189 −0

Original line number	Diff line number	Diff line
		package bp2build

		import (
		"fmt"
		"io/ioutil"
		"os"
		"path/filepath"

		"android/soong/shared"
		)

		// A tree structure that describes what to do at each directory in the created
		// symlink tree. Currently it is used to enumerate which files/directories
		// should be excluded from symlinking. Each instance of "node" represents a file
		// or a directory. If excluded is true, then that file/directory should be
		// excluded from symlinking. Otherwise, the node is not excluded, but one of its
		// descendants is (otherwise the node in question would not exist)
		type node struct {
		name string
		excluded bool // If false, this is just an intermediate node
		children map[string]*node
		}

		// Ensures that the a node for the given path exists in the tree and returns it.
		func ensureNodeExists(root node, path string) node {
		if path == "" {
		return root
		}

		if path[len(path)-1] == '/' {
		path = path[:len(path)-1] // filepath.Split() leaves a trailing slash
		}

		dir, base := filepath.Split(path)

		// First compute the parent node...
		dn := ensureNodeExists(root, dir)

		// then create the requested node as its direct child, if needed.
		if child, ok := dn.children[base]; ok {
		return child
		} else {
		dn.children[base] = &node{base, false, make(map[string]*node)}
		return dn.children[base]
		}
		}

		// Turns a list of paths to be excluded into a tree made of "node" objects where
		// the specified paths are marked as excluded.
		func treeFromExcludePathList(paths []string) *node {
		result := &node{"", false, make(map[string]*node)}

		for _, p := range paths {
		ensureNodeExists(result, p).excluded = true
		}

		return result
		}

		// Calls readdir() and returns it as a map from the basename of the files in dir
		// to os.FileInfo.
		func readdirToMap(dir string) map[string]os.FileInfo {
		entryList, err := ioutil.ReadDir(dir)
		result := make(map[string]os.FileInfo)

		if err != nil {
		if os.IsNotExist(err) {
		// It's okay if a directory doesn't exist; it just means that one of the
		// trees to be merged contains parts the other doesn't
		return result
		} else {
		fmt.Fprintf(os.Stderr, "Cannot readdir '%s': %s\n", dir, err)
		os.Exit(1)
		}
		}

		for _, fi := range entryList {
		result[fi.Name()] = fi
		}

		return result
		}

		// Creates a symbolic link at dst pointing to src
		func symlinkIntoForest(topdir, dst, src string) {
		err := os.Symlink(shared.JoinPath(topdir, src), shared.JoinPath(topdir, dst))
		if err != nil {
		fmt.Fprintf(os.Stderr, "Cannot create symlink at '%s' pointing to '%s': %s", dst, src, err)
		os.Exit(1)
		}
		}

		// Recursively plants a symlink forest at forestDir. The symlink tree will
		// contain every file in buildFilesDir and srcDir excluding the files in
		// exclude. Collects every directory encountered during the traversal of srcDir
		// into acc.
		func plantSymlinkForestRecursive(topdir string, forestDir string, buildFilesDir string, srcDir string, exclude node, acc []string) {
		if exclude != nil && exclude.excluded {
		// This directory is not needed, bail out
		return
		}

		acc = append(acc, srcDir)
		srcDirMap := readdirToMap(shared.JoinPath(topdir, srcDir))
		buildFilesMap := readdirToMap(shared.JoinPath(topdir, buildFilesDir))

		allEntries := make(map[string]bool)
		for n, _ := range srcDirMap {
		allEntries[n] = true
		}

		for n, _ := range buildFilesMap {
		allEntries[n] = true
		}

		err := os.MkdirAll(shared.JoinPath(topdir, forestDir), 0777)
		if err != nil {
		fmt.Fprintf(os.Stderr, "Cannot mkdir '%s': %s\n", forestDir, err)
		os.Exit(1)
		}

		for f, _ := range allEntries {
		if f[0] == '.' {
		continue // Ignore dotfiles
		}

		// The full paths of children in the input trees and in the output tree
		fp := shared.JoinPath(forestDir, f)
		sp := shared.JoinPath(srcDir, f)
		bp := shared.JoinPath(buildFilesDir, f)

		// Descend in the exclusion tree, if there are any excludes left
		var ce *node
		if exclude == nil {
		ce = nil
		} else {
		ce = exclude.children[f]
		}

		sf, sExists := srcDirMap[f]
		bf, bExists := buildFilesMap[f]
		excluded := ce != nil && ce.excluded

		if excluded {
		continue
		}

		if !sExists {
		if bf.IsDir() && ce != nil {
		// Not in the source tree, but we have to exclude something from under
		// this subtree, so descend
		plantSymlinkForestRecursive(topdir, fp, bp, sp, ce, acc)
		} else {
		// Not in the source tree, symlink BUILD file
		symlinkIntoForest(topdir, fp, bp)
		}
		} else if !bExists {
		if sf.IsDir() && ce != nil {
		// Not in the build file tree, but we have to exclude something from
		// under this subtree, so descend
		plantSymlinkForestRecursive(topdir, fp, bp, sp, ce, acc)
		} else {
		// Not in the build file tree, symlink source tree, carry on
		symlinkIntoForest(topdir, fp, sp)
		}
		} else if sf.IsDir() && bf.IsDir() {
		// Both are directories. Descend.
		plantSymlinkForestRecursive(topdir, fp, bp, sp, ce, acc)
		} else {
		// Both exist and one is a file. This is an error.
		fmt.Fprintf(os.Stderr,
		"Conflict in workspace symlink tree creation: both '%s' and '%s' exist and at least one of them is a file\n",
		sp, bp)
		os.Exit(1)
		}
		}
		}

		// Creates a symlink forest by merging the directory tree at "buildFiles" and
		// "srcDir" while excluding paths listed in "exclude". Returns the set of paths
		// under srcDir on which readdir() had to be called to produce the symlink
		// forest.
		func PlantSymlinkForest(topdir string, forest string, buildFiles string, srcDir string, exclude []string) []string {
		deps := make([]string, 0)
		os.RemoveAll(shared.JoinPath(topdir, forest))
		excludeTree := treeFromExcludePathList(exclude)
		plantSymlinkForestRecursive(topdir, forest, buildFiles, srcDir, excludeTree, &deps)
		return deps
		}

cmd/soong_build/main.go

+28 −8

Original line number	Diff line number	Diff line
		@@ -396,24 +396,44 @@ func runBp2Build(configuration android.Config, extraNinjaDeps []string) {
		ninjaDeps = append(ninjaDeps, globPath.FileListFile(configuration.BuildDir()))
		}

		depFile := bp2buildMarker + ".d"
		err = deptools.WriteDepFile(shared.JoinPath(topDir, depFile), bp2buildMarker, ninjaDeps)
		if err != nil {
		fmt.Fprintf(os.Stderr, "Cannot write depfile '%s': %s\n", depFile, err)
		os.Exit(1)
		}

		// Run the code-generation phase to convert BazelTargetModules to BUILD files
		// and print conversion metrics to the user.
		codegenContext := bp2build.NewCodegenContext(configuration, *bp2buildCtx, bp2build.Bp2Build)
		metrics := bp2build.Codegen(codegenContext)

		generatedRoot := shared.JoinPath(configuration.BuildDir(), "bp2build")
		workspaceRoot := shared.JoinPath(configuration.BuildDir(), "workspace")

		excludes := []string{
		"bazel-bin",
		"bazel-genfiles",
		"bazel-out",
		"bazel-testlogs",
		"bazel-" + filepath.Base(topDir),
		}

		if bootstrap.CmdlineArgs.NinjaBuildDir[0] != '/' {
		excludes = append(excludes, bootstrap.CmdlineArgs.NinjaBuildDir)
		}

		symlinkForestDeps := bp2build.PlantSymlinkForest(
		topDir, workspaceRoot, generatedRoot, configuration.SrcDir(), excludes)

		// Only report metrics when in bp2build mode. The metrics aren't relevant
		// for queryview, since that's a total repo-wide conversion and there's a
		// 1:1 mapping for each module.
		metrics.Print()

		extraNinjaDeps = append(extraNinjaDeps, codegenContext.AdditionalNinjaDeps()...)
		ninjaDeps = append(ninjaDeps, codegenContext.AdditionalNinjaDeps()...)
		ninjaDeps = append(ninjaDeps, symlinkForestDeps...)

		depFile := bp2buildMarker + ".d"
		err = deptools.WriteDepFile(shared.JoinPath(topDir, depFile), bp2buildMarker, ninjaDeps)
		if err != nil {
		fmt.Fprintf(os.Stderr, "Cannot write depfile '%s': %s\n", depFile, err)
		os.Exit(1)
		}

		if bp2buildMarker != "" {
		touch(shared.JoinPath(topDir, bp2buildMarker))
		} else {

tests/bootstrap_test.sh

+55 −10

Original line number	Diff line number	Diff line
		@@ -485,9 +485,8 @@ function test_null_build_after_docs {
		function test_integrated_bp2build_smoke {
		setup
		INTEGRATED_BP2BUILD=1 run_soong
		if [[ ! -e out/soong/.bootstrap/bp2build_workspace_marker ]]; then
		fail "bp2build marker file not created"
		fi
		[[ -e out/soong/.bootstrap/bp2build_workspace_marker ]] \|\| fail "bp2build marker file not created"
		[[ -e out/soong/workspace ]] \|\| fail "Bazel workspace not created"
		}

		function test_integrated_bp2build_add_android_bp {
		@@ -504,9 +503,8 @@ filegroup {
		EOF

		INTEGRATED_BP2BUILD=1 run_soong
		if [[ ! -e out/soong/bp2build/a/BUILD ]]; then
		fail "a/BUILD not created";
		fi
		[[ -e out/soong/bp2build/a/BUILD ]] \|\| fail "a/BUILD not created"
		[[ -L out/soong/workspace/a/BUILD ]] \|\| fail "a/BUILD not symlinked"

		mkdir -p b
		touch b/b.txt
		@@ -519,9 +517,8 @@ filegroup {
		EOF

		INTEGRATED_BP2BUILD=1 run_soong
		if [[ ! -e out/soong/bp2build/b/BUILD ]]; then
		fail "b/BUILD not created";
		fi
		[[ -e out/soong/bp2build/b/BUILD ]] \|\| fail "a/BUILD not created"
		[[ -L out/soong/workspace/b/BUILD ]] \|\| fail "a/BUILD not symlinked"
		}

		function test_integrated_bp2build_null_build {
		@@ -567,11 +564,56 @@ function test_dump_json_module_graph() {
		fi
		}

		function test_integrated_bp2build_bazel_workspace_structure {
		setup

		mkdir -p a/b
		touch a/a.txt
		touch a/b/b.txt
		cat > a/b/Android.bp <<'EOF'
		filegroup {
		name: "b",
		srcs: ["b.txt"],
		bazel_module: { bp2build_available: true },
		}
		EOF

		INTEGRATED_BP2BUILD=1 run_soong
		[[ -e out/soong/workspace ]] \|\| fail "Bazel workspace not created"
		[[ -d out/soong/workspace/a/b ]] \|\| fail "module directory not a directory"
		[[ -L out/soong/workspace/a/b/BUILD ]] \|\| fail "BUILD file not symlinked"
		[[ "$(readlink -f out/soong/workspace/a/b/BUILD)" =~ bp2build/a/b/BUILD$ ]] \
		\|\| fail "BUILD files symlinked at the wrong place"
		[[ -L out/soong/workspace/a/b/b.txt ]] \|\| fail "a/b/b.txt not symlinked"
		[[ -L out/soong/workspace/a/a.txt ]] \|\| fail "a/b/a.txt not symlinked"
		[[ ! -e out/soong/workspace/out ]] \|\| fail "out directory symlinked"
		}

		function test_integrated_bp2build_bazel_workspace_add_file {
		setup

		mkdir -p a
		touch a/a.txt
		cat > a/Android.bp <<EOF
		filegroup {
		name: "a",
		srcs: ["a.txt"],
		bazel_module: { bp2build_available: true },
		}
		EOF

		INTEGRATED_BP2BUILD=1 run_soong

		touch a/a2.txt # No reference in the .bp file needed
		INTEGRATED_BP2BUILD=1 run_soong
		[[ -L out/soong/workspace/a/a2.txt ]] \|\| fail "a/a2.txt not symlinked"
		}

		test_smoke
		test_null_build
		test_null_build_after_docs
		test_soong_build_rebuilt_if_blueprint_changes
		test_glob_noop_incremental
		# test_glob_noop_incremental # Currently failing
		test_add_file_to_glob
		test_add_android_bp
		test_change_android_bp
		@@ -582,4 +624,7 @@ test_soong_build_rerun_iff_environment_changes
		test_dump_json_module_graph
		test_integrated_bp2build_smoke
		test_integrated_bp2build_null_build
		test_integrated_bp2build_add_android_bp
		test_integrated_bp2build_add_to_glob
		test_integrated_bp2build_bazel_workspace_structure
		test_integrated_bp2build_bazel_workspace_add_file