Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit b353cca4 authored by Lukacs T. Berki's avatar Lukacs T. Berki
Browse files

Create a synthetic Bazel workspace. 

It's under out/soong/workspace and is a symlink forest that merges BUILD
files generated by bp2build and the source tree.

bazel-* symlinks and $OUTDIR are currently ignored.

Test: Presubmits.
Change-Id: If14df23bd0f6a6f2d7d1140f95213f2aee66d652
parent 5fb28ffd

bp2build/Android.bp

+1 −0
Original line number Diff line number Diff line
@@ -14,6 +14,7 @@ bootstrap_go_package { 
        "constants.go",

        "conversion.go",

        "metrics.go",

        "symlink_forest.go",

    ],

    deps: [

        "soong-android",

bp2build/conversion_test.go

+2 −2
Original line number Diff line number Diff line
@@ -19,14 +19,14 @@ import ( 
	"testing"

)



type filepath struct {

type bazelFilepath struct {

	dir      string

	basename string

}



func TestCreateBazelFiles_QueryView_AddsTopLevelFiles(t *testing.T) {

	files := CreateBazelFiles(map[string]RuleShim{}, map[string]BazelTargets{}, QueryView)

	expectedFilePaths := []filepath{

	expectedFilePaths := []bazelFilepath{

		{

			dir:      "",

			basename: "BUILD",

bp2build/symlink_forest.go

0 → 100644
+189 −0
Original line number Diff line number Diff line 
package bp2build



import (

	"fmt"

	"io/ioutil"

	"os"

	"path/filepath"



	"android/soong/shared"

)



// A tree structure that describes what to do at each directory in the created

// symlink tree. Currently it is used to enumerate which files/directories

// should be excluded from symlinking. Each instance of "node" represents a file

// or a directory. If excluded is true, then that file/directory should be

// excluded from symlinking. Otherwise, the node is not excluded, but one of its

// descendants is (otherwise the node in question would not exist)

type node struct {

	name     string

	excluded bool // If false, this is just an intermediate node

	children map[string]*node

}



// Ensures that the a node for the given path exists in the tree and returns it.

func ensureNodeExists(root *node, path string) *node {

	if path == "" {

		return root

	}



	if path[len(path)-1] == '/' {

		path = path[:len(path)-1] // filepath.Split() leaves a trailing slash

	}



	dir, base := filepath.Split(path)



	// First compute the parent node...

	dn := ensureNodeExists(root, dir)



	// then create the requested node as its direct child, if needed.

	if child, ok := dn.children[base]; ok {

		return child

	} else {

		dn.children[base] = &node{base, false, make(map[string]*node)}

		return dn.children[base]

	}

}



// Turns a list of paths to be excluded into a tree made of "node" objects where

// the specified paths are marked as excluded.

func treeFromExcludePathList(paths []string) *node {

	result := &node{"", false, make(map[string]*node)}



	for _, p := range paths {

		ensureNodeExists(result, p).excluded = true

	}



	return result

}



// Calls readdir() and returns it as a map from the basename of the files in dir

// to os.FileInfo.

func readdirToMap(dir string) map[string]os.FileInfo {

	entryList, err := ioutil.ReadDir(dir)

	result := make(map[string]os.FileInfo)



	if err != nil {

		if os.IsNotExist(err) {

			// It's okay if a directory doesn't exist; it just means that one of the

			// trees to be merged contains parts the other doesn't

			return result

		} else {

			fmt.Fprintf(os.Stderr, "Cannot readdir '%s': %s\n", dir, err)

			os.Exit(1)

		}

	}



	for _, fi := range entryList {

		result[fi.Name()] = fi

	}



	return result

}



// Creates a symbolic link at dst pointing to src

func symlinkIntoForest(topdir, dst, src string) {

	err := os.Symlink(shared.JoinPath(topdir, src), shared.JoinPath(topdir, dst))

	if err != nil {

		fmt.Fprintf(os.Stderr, "Cannot create symlink at '%s' pointing to '%s': %s", dst, src, err)

		os.Exit(1)

	}

}



// Recursively plants a symlink forest at forestDir. The symlink tree will

// contain every file in buildFilesDir and srcDir excluding the files in

// exclude. Collects every directory encountered during the traversal of srcDir

// into acc.

func plantSymlinkForestRecursive(topdir string, forestDir string, buildFilesDir string, srcDir string, exclude *node, acc *[]string) {

	if exclude != nil && exclude.excluded {

		// This directory is not needed, bail out

		return

	}



	*acc = append(*acc, srcDir)

	srcDirMap := readdirToMap(shared.JoinPath(topdir, srcDir))

	buildFilesMap := readdirToMap(shared.JoinPath(topdir, buildFilesDir))



	allEntries := make(map[string]bool)

	for n, _ := range srcDirMap {

		allEntries[n] = true

	}



	for n, _ := range buildFilesMap {

		allEntries[n] = true

	}



	err := os.MkdirAll(shared.JoinPath(topdir, forestDir), 0777)

	if err != nil {

		fmt.Fprintf(os.Stderr, "Cannot mkdir '%s': %s\n", forestDir, err)

		os.Exit(1)

	}



	for f, _ := range allEntries {

		if f[0] == '.' {

			continue // Ignore dotfiles

		}



		// The full paths of children in the input trees and in the output tree

		fp := shared.JoinPath(forestDir, f)

		sp := shared.JoinPath(srcDir, f)

		bp := shared.JoinPath(buildFilesDir, f)



		// Descend in the exclusion tree, if there are any excludes left

		var ce *node

		if exclude == nil {

			ce = nil

		} else {

			ce = exclude.children[f]

		}



		sf, sExists := srcDirMap[f]

		bf, bExists := buildFilesMap[f]

		excluded := ce != nil && ce.excluded



		if excluded {

			continue

		}



		if !sExists {

			if bf.IsDir() && ce != nil {

				// Not in the source tree, but we have to exclude something from under

				// this subtree, so descend

				plantSymlinkForestRecursive(topdir, fp, bp, sp, ce, acc)

			} else {

				// Not in the source tree, symlink BUILD file

				symlinkIntoForest(topdir, fp, bp)

			}

		} else if !bExists {

			if sf.IsDir() && ce != nil {

				// Not in the build file tree, but we have to exclude something from

				// under this subtree, so descend

				plantSymlinkForestRecursive(topdir, fp, bp, sp, ce, acc)

			} else {

				// Not in the build file tree, symlink source tree, carry on

				symlinkIntoForest(topdir, fp, sp)

			}

		} else if sf.IsDir() && bf.IsDir() {

			// Both are directories. Descend.

			plantSymlinkForestRecursive(topdir, fp, bp, sp, ce, acc)

		} else {

			// Both exist and one is a file. This is an error.

			fmt.Fprintf(os.Stderr,

				"Conflict in workspace symlink tree creation: both '%s' and '%s' exist and at least one of them is a file\n",

				sp, bp)

			os.Exit(1)

		}

	}

}



// Creates a symlink forest by merging the directory tree at "buildFiles" and

// "srcDir" while excluding paths listed in "exclude". Returns the set of paths

// under srcDir on which readdir() had to be called to produce the symlink

// forest.

func PlantSymlinkForest(topdir string, forest string, buildFiles string, srcDir string, exclude []string) []string {

	deps := make([]string, 0)

	os.RemoveAll(shared.JoinPath(topdir, forest))

	excludeTree := treeFromExcludePathList(exclude)

	plantSymlinkForestRecursive(topdir, forest, buildFiles, srcDir, excludeTree, &deps)

	return deps

}

cmd/soong_build/main.go

+28 −8
Original line number Diff line number Diff line
@@ -396,24 +396,44 @@ func runBp2Build(configuration android.Config, extraNinjaDeps []string) { 
		ninjaDeps = append(ninjaDeps, globPath.FileListFile(configuration.BuildDir()))

	}



	depFile := bp2buildMarker + ".d"

	err = deptools.WriteDepFile(shared.JoinPath(topDir, depFile), bp2buildMarker, ninjaDeps)

	if err != nil {

		fmt.Fprintf(os.Stderr, "Cannot write depfile '%s': %s\n", depFile, err)

		os.Exit(1)

	}



	// Run the code-generation phase to convert BazelTargetModules to BUILD files

	// and print conversion metrics to the user.

	codegenContext := bp2build.NewCodegenContext(configuration, *bp2buildCtx, bp2build.Bp2Build)

	metrics := bp2build.Codegen(codegenContext)



	generatedRoot := shared.JoinPath(configuration.BuildDir(), "bp2build")

	workspaceRoot := shared.JoinPath(configuration.BuildDir(), "workspace")



	excludes := []string{

		"bazel-bin",

		"bazel-genfiles",

		"bazel-out",

		"bazel-testlogs",

		"bazel-" + filepath.Base(topDir),

	}



	if bootstrap.CmdlineArgs.NinjaBuildDir[0] != '/' {

		excludes = append(excludes, bootstrap.CmdlineArgs.NinjaBuildDir)

	}



	symlinkForestDeps := bp2build.PlantSymlinkForest(

		topDir, workspaceRoot, generatedRoot, configuration.SrcDir(), excludes)



	// Only report metrics when in bp2build mode. The metrics aren't relevant

	// for queryview, since that's a total repo-wide conversion and there's a

	// 1:1 mapping for each module.

	metrics.Print()



	extraNinjaDeps = append(extraNinjaDeps, codegenContext.AdditionalNinjaDeps()...)

	ninjaDeps = append(ninjaDeps, codegenContext.AdditionalNinjaDeps()...)

	ninjaDeps = append(ninjaDeps, symlinkForestDeps...)



	depFile := bp2buildMarker + ".d"

	err = deptools.WriteDepFile(shared.JoinPath(topDir, depFile), bp2buildMarker, ninjaDeps)

	if err != nil {

		fmt.Fprintf(os.Stderr, "Cannot write depfile '%s': %s\n", depFile, err)

		os.Exit(1)

	}



	if bp2buildMarker != "" {

		touch(shared.JoinPath(topDir, bp2buildMarker))

	} else {

tests/bootstrap_test.sh

+55 −10
Original line number Diff line number Diff line
@@ -485,9 +485,8 @@ function test_null_build_after_docs { 
function test_integrated_bp2build_smoke {

  setup

  INTEGRATED_BP2BUILD=1 run_soong

  if [[ ! -e out/soong/.bootstrap/bp2build_workspace_marker ]]; then

    fail "bp2build marker file not created"

  fi

  [[ -e out/soong/.bootstrap/bp2build_workspace_marker ]] || fail "bp2build marker file not created"

  [[ -e out/soong/workspace ]] || fail "Bazel workspace not created"

}



function test_integrated_bp2build_add_android_bp {
@@ -504,9 +503,8 @@ filegroup { 
EOF



  INTEGRATED_BP2BUILD=1 run_soong

  if [[ ! -e out/soong/bp2build/a/BUILD ]]; then

    fail "a/BUILD not created";

  fi

  [[ -e out/soong/bp2build/a/BUILD ]] || fail "a/BUILD not created"

  [[ -L out/soong/workspace/a/BUILD ]] || fail "a/BUILD not symlinked"



  mkdir -p b

  touch b/b.txt
@@ -519,9 +517,8 @@ filegroup { 
EOF



  INTEGRATED_BP2BUILD=1 run_soong

  if [[ ! -e out/soong/bp2build/b/BUILD ]]; then

    fail "b/BUILD not created";

  fi

  [[ -e out/soong/bp2build/b/BUILD ]] || fail "a/BUILD not created"

  [[ -L out/soong/workspace/b/BUILD ]] || fail "a/BUILD not symlinked"

}



function test_integrated_bp2build_null_build {
@@ -567,11 +564,56 @@ function test_dump_json_module_graph() { 
  fi

}



function test_integrated_bp2build_bazel_workspace_structure {

  setup



  mkdir -p a/b

  touch a/a.txt

  touch a/b/b.txt

  cat > a/b/Android.bp <<'EOF'

filegroup {

  name: "b",

  srcs: ["b.txt"],

  bazel_module: { bp2build_available: true },

}

EOF



  INTEGRATED_BP2BUILD=1 run_soong

  [[ -e out/soong/workspace ]] || fail "Bazel workspace not created"

  [[ -d out/soong/workspace/a/b ]] || fail "module directory not a directory"

  [[ -L out/soong/workspace/a/b/BUILD ]] || fail "BUILD file not symlinked"

  [[ "$(readlink -f out/soong/workspace/a/b/BUILD)" =~ bp2build/a/b/BUILD$ ]] \

    || fail "BUILD files symlinked at the wrong place"

  [[ -L out/soong/workspace/a/b/b.txt ]] || fail "a/b/b.txt not symlinked"

  [[ -L out/soong/workspace/a/a.txt ]] || fail "a/b/a.txt not symlinked"

  [[ ! -e out/soong/workspace/out ]] || fail "out directory symlinked"

}



function test_integrated_bp2build_bazel_workspace_add_file {

  setup



  mkdir -p a

  touch a/a.txt

  cat > a/Android.bp <<EOF

filegroup {

  name: "a",

  srcs: ["a.txt"],

  bazel_module: { bp2build_available: true },

}

EOF



  INTEGRATED_BP2BUILD=1 run_soong



  touch a/a2.txt  # No reference in the .bp file needed

  INTEGRATED_BP2BUILD=1 run_soong

  [[ -L out/soong/workspace/a/a2.txt ]] || fail "a/a2.txt not symlinked"

}



test_smoke

test_null_build

test_null_build_after_docs

test_soong_build_rebuilt_if_blueprint_changes

test_glob_noop_incremental

# test_glob_noop_incremental  # Currently failing

test_add_file_to_glob

test_add_android_bp

test_change_android_bp
@@ -582,4 +624,7 @@ test_soong_build_rerun_iff_environment_changes 
test_dump_json_module_graph

test_integrated_bp2build_smoke

test_integrated_bp2build_null_build

test_integrated_bp2build_add_android_bp

test_integrated_bp2build_add_to_glob

test_integrated_bp2build_bazel_workspace_structure

test_integrated_bp2build_bazel_workspace_add_file