Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit e03bb610 authored by Rupert Shuttleworth's avatar Rupert Shuttleworth
Browse files

Use Finder for getPathsToIgnoredBuildFiles() to try and speed up builds with slow hard drives. 

NOTE: This relies on https://chromium-review.googlesource.com/c/angle/angle/+/2906252 being merged into AOSP first

Test: bazel build //bionic/... //external/... //frameworks/... //system/...

Change-Id: Id552941274ffbc9955dfcd8f545a58f1e3368845
parent cf21e25e

cmd/soong_build/main.go

+41 −40
Original line number Diff line number Diff line
@@ -17,7 +17,6 @@ package main 
import (

	"flag"

	"fmt"

	"io/fs"

	"io/ioutil"

	"os"

	"path/filepath"
@@ -364,54 +363,41 @@ func touch(path string) { 
// - won't be overwritten by corresponding bp2build generated files

//

// And return their paths so they can be left out of the Bazel workspace dir (i.e. ignored)

func getPathsToIgnoredBuildFiles(topDir string, outDir string, generatedRoot string) ([]string, error) {

func getPathsToIgnoredBuildFiles(topDir string, generatedRoot string, srcDirBazelFiles []string) []string {

	paths := make([]string, 0)



	err := filepath.WalkDir(topDir, func(fFullPath string, fDirEntry fs.DirEntry, err error) error {

	for _, srcDirBazelFileRelativePath := range srcDirBazelFiles {

		srcDirBazelFileFullPath := shared.JoinPath(topDir, srcDirBazelFileRelativePath)

		fileInfo, err := os.Stat(srcDirBazelFileFullPath)

		if err != nil {

			// Warn about error, but continue trying to walk the directory tree

			fmt.Fprintf(os.Stderr, "WARNING: Error accessing path '%s', err: %s\n", fFullPath, err)

			return nil

			// Warn about error, but continue trying to check files

			fmt.Fprintf(os.Stderr, "WARNING: Error accessing path '%s', err: %s\n", srcDirBazelFileFullPath, err)

			continue

		}

		if fDirEntry.IsDir() {

		if fileInfo.IsDir() {

			// Don't ignore entire directories

			return nil

			continue

		}

		if !(fDirEntry.Name() == "BUILD" || fDirEntry.Name() == "BUILD.bazel") {

		if !(fileInfo.Name() == "BUILD" || fileInfo.Name() == "BUILD.bazel") {

			// Don't ignore this file - it is not a build file

			return nil

			continue

		}

		f := strings.TrimPrefix(fFullPath, topDir+"/")

		if strings.HasPrefix(f, ".repo/") {

			// Don't check for files to ignore in the .repo dir (recursively)

			return fs.SkipDir

		}

		if strings.HasPrefix(f, outDir+"/") {

			// Don't check for files to ignore in the out dir (recursively)

			return fs.SkipDir

		}

		if strings.HasPrefix(f, generatedRoot) {

			// Don't check for files to ignore in the bp2build dir (recursively)

			// NOTE: This is usually under outDir

			return fs.SkipDir

		}

		fDir := filepath.Dir(f)

		if android.ShouldKeepExistingBuildFileForDir(fDir) {

		srcDirBazelFileDir := filepath.Dir(srcDirBazelFileRelativePath)

		if android.ShouldKeepExistingBuildFileForDir(srcDirBazelFileDir) {

			// Don't ignore this existing build file

			return nil

			continue

		}

		f_bp2build := shared.JoinPath(topDir, generatedRoot, f)

		if _, err := os.Stat(f_bp2build); err == nil {

		correspondingBp2BuildFile := shared.JoinPath(topDir, generatedRoot, srcDirBazelFileRelativePath)

		if _, err := os.Stat(correspondingBp2BuildFile); err == nil {

			// If bp2build generated an alternate BUILD file, don't exclude this workspace path

			// BUILD file clash resolution happens later in the symlink forest creation

			return nil

			continue

		}

		fmt.Fprintf(os.Stderr, "Ignoring existing BUILD file: %s\n", srcDirBazelFileRelativePath)

		paths = append(paths, srcDirBazelFileRelativePath)

	}

		fmt.Fprintf(os.Stderr, "Ignoring existing BUILD file: %s\n", f)

		paths = append(paths, f)

		return nil

	})



	return paths, err

	return paths

}



// Returns temporary symlink forest excludes necessary for bazel build //external/... (and bazel build //frameworks/...) to work
@@ -431,6 +417,22 @@ func getTemporaryExcludes() []string { 
	return excludes

}



// Read the bazel.list file that the Soong Finder already dumped earlier (hopefully)

// It contains the locations of BUILD files, BUILD.bazel files, etc. in the source dir

func getExistingBazelRelatedFiles(topDir string) ([]string, error) {

	bazelFinderFile := filepath.Join(filepath.Dir(bootstrap.CmdlineArgs.ModuleListFile), "bazel.list")

	if !filepath.IsAbs(bazelFinderFile) {

		// Assume this was a relative path under topDir

		bazelFinderFile = filepath.Join(topDir, bazelFinderFile)

	}

	data, err := ioutil.ReadFile(bazelFinderFile)

	if err != nil {

		return nil, err

	}

	files := strings.Split(strings.TrimSpace(string(data)), "\n")

	return files, nil

}



// Run Soong in the bp2build mode. This creates a standalone context that registers

// an alternate pipeline of mutators and singletons specifically for generating

// Bazel BUILD files instead of Ninja files.
@@ -489,14 +491,13 @@ func runBp2Build(configuration android.Config, extraNinjaDeps []string) { 
		excludes = append(excludes, bootstrap.CmdlineArgs.NinjaBuildDir)

	}



	// FIXME: Don't hardcode this here

	topLevelOutDir := "out"



	pathsToIgnoredBuildFiles, err := getPathsToIgnoredBuildFiles(topDir, topLevelOutDir, generatedRoot)

	existingBazelRelatedFiles, err := getExistingBazelRelatedFiles(topDir)

	if err != nil {

		fmt.Fprintf(os.Stderr, "Error walking SrcDir: '%s': %s\n", configuration.SrcDir(), err)

		fmt.Fprintf(os.Stderr, "Error determining existing Bazel-related files: %s\n", err)

		os.Exit(1)

	}



	pathsToIgnoredBuildFiles := getPathsToIgnoredBuildFiles(topDir, generatedRoot, existingBazelRelatedFiles)

	excludes = append(excludes, pathsToIgnoredBuildFiles...)



	excludes = append(excludes, getTemporaryExcludes()...)