Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 0096079b authored by Rupert Shuttleworth's avatar Rupert Shuttleworth
Browse files

Make bazel build //external/... work by only allowing "approved" existing... 

Make bazel build //external/... work by only allowing "approved" existing BUILD files in the Bazel workspace.

Test: bazel build //bionic/... //external/... //frameworks/... //system/...

Test: ./build/bazel/scripts/run_presubmits.sh

Change-Id: I91865ca87c6535053e7a14d2526ff3ce0991bfea
parent 37ef2239

android/bazel.go

+32 −43
Original line number Diff line number Diff line
@@ -126,40 +126,21 @@ const ( 
)



var (

	// Do not write BUILD files for these directories

	// NOTE: this is not recursive

	bp2buildDoNotWriteBuildFileList = []string{

		// Don't generate these BUILD files - because external BUILD files already exist

		"external/boringssl",

		"external/brotli",

		"external/dagger2",

		"external/flatbuffers",

		"external/gflags",

		"external/google-fruit",

		"external/grpc-grpc",

		"external/grpc-grpc/test/core/util",

		"external/grpc-grpc/test/cpp/common",

		"external/grpc-grpc/third_party/address_sorting",

		"external/nanopb-c",

		"external/nos/host/generic",

		"external/nos/host/generic/libnos",

		"external/nos/host/generic/libnos/generator",

		"external/nos/host/generic/libnos_datagram",

		"external/nos/host/generic/libnos_transport",

		"external/nos/host/generic/nugget/proto",

		"external/perfetto",

		"external/protobuf",

		"external/rust/cxx",

		"external/rust/cxx/demo",

		"external/ruy",

		"external/tensorflow",

		"external/tensorflow/tensorflow/lite",

		"external/tensorflow/tensorflow/lite/java",

		"external/tensorflow/tensorflow/lite/kernels",

		"external/tflite-support",

		"external/tinyalsa_new",

		"external/wycheproof",

		"external/libyuv",

	// Keep any existing BUILD files (and do not generate new BUILD files) for these directories

	bp2buildKeepExistingBuildFile = map[string]bool{

		// This is actually build/bazel/build.BAZEL symlinked to ./BUILD

		".":/*recrusive = */ false,



		"build/bazel":/* recursive = */ true,

		"build/pesto":/* recursive = */ true,



		// external/bazelbuild-rules_android/... is needed by mixed builds, otherwise mixed builds analysis fails

		// e.g. ERROR: Analysis of target '@soong_injection//mixed_builds:buildroot' failed

		"external/bazelbuild-rules_android":/* recursive = */ true,



		"prebuilts/clang/host/linux-x86":/* recursive = */ false,

		"prebuilts/sdk":/* recursive = */ false,

		"prebuilts/sdk/tools":/* recursive = */ false,

	}



	// Configure modules in these directories to enable bp2build_available: true or false by default.
@@ -218,6 +199,7 @@ var ( 
		// libcxx

		"libBionicBenchmarksUtils", // cc_library_static, fatal error: 'map' file not found, from libcxx

		"fmtlib",                   // cc_library_static, fatal error: 'cassert' file not found, from libcxx

		"fmtlib_ndk",               // cc_library_static, fatal error: 'cassert' file not found

		"libbase",                  // http://b/186826479, cc_library, fatal error: 'memory' file not found, from libcxx



		// http://b/186024507: Includes errors because of the system_shared_libs default value.
@@ -229,6 +211,9 @@ var ( 
		"note_memtag_heap_async", // http://b/185127353: cc_library_static, error: feature.h not found

		"note_memtag_heap_sync",  // http://b/185127353: cc_library_static, error: feature.h not found



		"libjemalloc5",           // cc_library, ld.lld: error: undefined symbol: memset

		"gwp_asan_crash_handler", // cc_library, ld.lld: error: undefined symbol: memset



		// Tests. Handle later.

		"libbionic_tests_headers_posix", // http://b/186024507, cc_library_static, sched.h, time.h not found

		"libjemalloc5_integrationtest",
@@ -254,17 +239,12 @@ var ( 
	}



	// Used for quicker lookups

	bp2buildDoNotWriteBuildFile = map[string]bool{}

	bp2buildModuleDoNotConvert  = map[string]bool{}

	bp2buildCcLibraryStaticOnly = map[string]bool{}

	mixedBuildsDisabled         = map[string]bool{}

)



func init() {

	for _, moduleName := range bp2buildDoNotWriteBuildFileList {

		bp2buildDoNotWriteBuildFile[moduleName] = true

	}



	for _, moduleName := range bp2buildModuleDoNotConvertList {

		bp2buildModuleDoNotConvert[moduleName] = true

	}
@@ -282,13 +262,22 @@ func GenerateCcLibraryStaticOnly(ctx BazelConversionPathContext) bool { 
	return bp2buildCcLibraryStaticOnly[ctx.Module().Name()]

}



func ShouldWriteBuildFileForDir(dir string) bool {

	if _, ok := bp2buildDoNotWriteBuildFile[dir]; ok {

		return false

	} else {

func ShouldKeepExistingBuildFileForDir(dir string) bool {

	if _, ok := bp2buildKeepExistingBuildFile[dir]; ok {

		// Exact dir match

		return true

	}

	// Check if subtree match

	for prefix, recursive := range bp2buildKeepExistingBuildFile {

		if recursive {

			if strings.HasPrefix(dir, prefix+"/") {

				return true

			}

		}

	}

	// Default

	return false

}



// MixedBuildsEnabled checks that a module is ready to be replaced by a

// converted or handcrafted Bazel target.

bp2build/conversion.go

+1 −1
Original line number Diff line number Diff line
@@ -59,7 +59,7 @@ func CreateBazelFiles( 
func createBuildFiles(buildToTargets map[string]BazelTargets, mode CodegenMode) []BazelFile {

	files := make([]BazelFile, 0, len(buildToTargets))

	for _, dir := range android.SortedStringKeys(buildToTargets) {

		if mode == Bp2Build && !android.ShouldWriteBuildFileForDir(dir) {

		if mode == Bp2Build && android.ShouldKeepExistingBuildFileForDir(dir) {

			fmt.Printf("[bp2build] Not writing generated BUILD file for dir: '%s'\n", dir)

			continue

		}

cmd/soong_build/main.go

+87 −0
Original line number Diff line number Diff line
@@ -17,6 +17,7 @@ package main 
import (

	"flag"

	"fmt"

	"io/fs"

	"io/ioutil"

	"os"

	"path/filepath"
@@ -357,6 +358,80 @@ func touch(path string) { 
	}

}



// Find BUILD files in the srcDir which...

//

// - are not on the allow list (android/bazel.go#ShouldKeepExistingBuildFileForDir())

//

// - won't be overwritten by corresponding bp2build generated files

//

// And return their paths so they can be left out of the Bazel workspace dir (i.e. ignored)

func getPathsToIgnoredBuildFiles(topDir string, outDir string, generatedRoot string) ([]string, error) {

	paths := make([]string, 0)



	err := filepath.WalkDir(topDir, func(fFullPath string, fDirEntry fs.DirEntry, err error) error {

		if err != nil {

			// Warn about error, but continue trying to walk the directory tree

			fmt.Fprintf(os.Stderr, "WARNING: Error accessing path '%s', err: %s\n", fFullPath, err)

			return nil

		}

		if fDirEntry.IsDir() {

			// Don't ignore entire directories

			return nil

		}

		if !(fDirEntry.Name() == "BUILD" || fDirEntry.Name() == "BUILD.bazel") {

			// Don't ignore this file - it is not a build file

			return nil

		}

		f := strings.TrimPrefix(fFullPath, topDir+"/")

		if strings.HasPrefix(f, ".repo/") {

			// Don't check for files to ignore in the .repo dir (recursively)

			return fs.SkipDir

		}

		if strings.HasPrefix(f, outDir+"/") {

			// Don't check for files to ignore in the out dir (recursively)

			return fs.SkipDir

		}

		if strings.HasPrefix(f, generatedRoot) {

			// Don't check for files to ignore in the bp2build dir (recursively)

			// NOTE: This is usually under outDir

			return fs.SkipDir

		}

		fDir := filepath.Dir(f)

		if android.ShouldKeepExistingBuildFileForDir(fDir) {

			// Don't ignore this existing build file

			return nil

		}

		f_bp2build := shared.JoinPath(topDir, generatedRoot, f)

		if _, err := os.Stat(f_bp2build); err == nil {

			// If bp2build generated an alternate BUILD file, don't exclude this workspace path

			// BUILD file clash resolution happens later in the symlink forest creation

			return nil

		}

		fmt.Fprintf(os.Stderr, "Ignoring existing BUILD file: %s\n", f)

		paths = append(paths, f)

		return nil

	})



	return paths, err

}



// Returns temporary symlink forest excludes necessary for bazel build //external/... (and bazel build //frameworks/...) to work

func getTemporaryExcludes() []string {

	excludes := make([]string, 0)



	// FIXME: 'autotest_lib' is a symlink back to external/autotest, and this causes an infinite symlink expansion error for Bazel

	excludes = append(excludes, "external/autotest/venv/autotest_lib")



	// FIXME: The external/google-fruit/extras/bazel_root/third_party/fruit dir is poison

	// It contains several symlinks back to real source dirs, and those source dirs contain BUILD files we want to ignore

	excludes = append(excludes, "external/google-fruit/extras/bazel_root/third_party/fruit")



	// FIXME: 'frameworks/compile/slang' has a filegroup error due to an escaping issue

	excludes = append(excludes, "frameworks/compile/slang")



	return excludes

}



// Run Soong in the bp2build mode. This creates a standalone context that registers

// an alternate pipeline of mutators and singletons specifically for generating

// Bazel BUILD files instead of Ninja files.
@@ -415,6 +490,18 @@ func runBp2Build(configuration android.Config, extraNinjaDeps []string) { 
		excludes = append(excludes, bootstrap.CmdlineArgs.NinjaBuildDir)

	}



	// FIXME: Don't hardcode this here

	topLevelOutDir := "out"



	pathsToIgnoredBuildFiles, err := getPathsToIgnoredBuildFiles(topDir, topLevelOutDir, generatedRoot)

	if err != nil {

		fmt.Fprintf(os.Stderr, "Error walking SrcDir: '%s': %s\n", configuration.SrcDir(), err)

		os.Exit(1)

	}

	excludes = append(excludes, pathsToIgnoredBuildFiles...)



	excludes = append(excludes, getTemporaryExcludes()...)



	symlinkForestDeps := bp2build.PlantSymlinkForest(

		topDir, workspaceRoot, generatedRoot, configuration.SrcDir(), excludes)