Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 520e88b7 authored by Chris Parsons's avatar Chris Parsons
Browse files

Avoid deleting bp2build directory 

With this CL, we avoid deleting the bp2build directory and regenerating
all BUILD files. Instead, we regenerate BUILD files which have changes
on the filesystem, and delete old BUILD files which should no longer
exist.

This improves incremental Bazel server performance by about ~5 seconds.
Previously, Bazel would have cache misses for regenerated packages, and
would thus need to analyze them even if no meaningful changes took
place.

For ease of implementation, we avoid removing stale (empty) directories
from the bp2build workspace, but this should have no effect on the
build. (These will be removed on next clean)

Test: New integration tests
Test: Manual benchmarking in conjunction with persistent bazel server
Change-Id: I3e489ff403be34040122876012329060a2506366
parent 0897df14

bp2build/bp2build.go

+45 −4
Original line number Diff line number Diff line
@@ -17,21 +17,57 @@ package bp2build 
import (

	"fmt"

	"os"

	"path/filepath"

	"strings"



	"android/soong/android"

	"android/soong/bazel"

	"android/soong/shared"

)



func deleteFilesExcept(ctx *CodegenContext, rootOutputPath android.OutputPath, except []BazelFile) {

	// Delete files that should no longer be present.

	bp2buildDirAbs := shared.JoinPath(ctx.topDir, rootOutputPath.String())



	filesToDelete := make(map[string]struct{})

	err := filepath.Walk(bp2buildDirAbs,

		func(path string, info os.FileInfo, err error) error {

			if err != nil {

				return err

			}

			if !info.IsDir() {

				relPath, err := filepath.Rel(bp2buildDirAbs, path)

				if err != nil {

					return err

				}

				filesToDelete[relPath] = struct{}{}

			}

			return nil

		})

	if err != nil {

		fmt.Printf("ERROR reading %s: %s", bp2buildDirAbs, err)

		os.Exit(1)

	}



	for _, bazelFile := range except {

		filePath := filepath.Join(bazelFile.Dir, bazelFile.Basename)

		delete(filesToDelete, filePath)

	}

	for f, _ := range filesToDelete {

		absPath := shared.JoinPath(bp2buildDirAbs, f)

		if err := os.RemoveAll(absPath); err != nil {

			fmt.Printf("ERROR deleting %s: %s", absPath, err)

			os.Exit(1)

		}

	}

}



// Codegen is the backend of bp2build. The code generator is responsible for

// writing .bzl files that are equivalent to Android.bp files that are capable

// of being built with Bazel.

func Codegen(ctx *CodegenContext) *CodegenMetrics {

	// This directory stores BUILD files that could be eventually checked-in.

	bp2buildDir := android.PathForOutput(ctx, "bp2build")

	if err := android.RemoveAllOutputDir(bp2buildDir); err != nil {

		fmt.Printf("ERROR: Encountered error while cleaning %s: %s", bp2buildDir, err.Error())

	}



	res, errs := GenerateBazelTargets(ctx, true)

	if len(errs) > 0 {
@@ -44,6 +80,12 @@ func Codegen(ctx *CodegenContext) *CodegenMetrics { 
	}

	bp2buildFiles := CreateBazelFiles(ctx.Config(), nil, res.buildFileToTargets, ctx.mode)

	writeFiles(ctx, bp2buildDir, bp2buildFiles)

	// Delete files under the bp2build root which weren't just written. An

	// alternative would have been to delete the whole directory and write these

	// files. However, this would regenerate files which were otherwise unchanged

	// since the last bp2build run, which would have negative incremental

	// performance implications.

	deleteFilesExcept(ctx, bp2buildDir, bp2buildFiles)



	injectionFiles, err := CreateSoongInjectionDirFiles(ctx, res.metrics)

	if err != nil {
@@ -51,7 +93,6 @@ func Codegen(ctx *CodegenContext) *CodegenMetrics { 
		os.Exit(1)

	}

	writeFiles(ctx, android.PathForOutput(ctx, bazel.SoongInjectionDirName), injectionFiles)



	return &res.metrics

}

tests/bp2build_bazel_test.sh

+62 −0
Original line number Diff line number Diff line
@@ -21,6 +21,68 @@ function test_bp2build_null_build { 
  fi

}



# Tests that, if bp2build reruns due to a blueprint file changing, that

# BUILD files whose contents are unchanged are not regenerated.

function test_bp2build_unchanged {

  setup



  mkdir -p pkg

  touch pkg/x.txt

  cat > pkg/Android.bp <<'EOF'

filegroup {

    name: "x",

    srcs: ["x.txt"],

    bazel_module: {bp2build_available: true},

  }

EOF



  run_soong bp2build

  local -r buildfile_mtime1=$(stat -c "%y" out/soong/bp2build/pkg/BUILD.bazel)

  local -r marker_mtime1=$(stat -c "%y" out/soong/bp2build_workspace_marker)



  # Force bp2build to rerun by updating the timestamp of a blueprint file.

  touch pkg/Android.bp



  run_soong bp2build

  local -r buildfile_mtime2=$(stat -c "%y" out/soong/bp2build/pkg/BUILD.bazel)

  local -r marker_mtime2=$(stat -c "%y" out/soong/bp2build_workspace_marker)



  if [[ "$marker_mtime1" == "$marker_mtime2" ]]; then

    fail "Expected bp2build marker file to change"

  fi

  if [[ "$buildfile_mtime1" != "$buildfile_mtime2" ]]; then

    fail "BUILD.bazel was updated even though contents are same"

  fi

}



# Tests that blueprint files that are deleted are not present when the

# bp2build tree is regenerated.

function test_bp2build_deleted_blueprint {

  setup



  mkdir -p pkg

  touch pkg/x.txt

  cat > pkg/Android.bp <<'EOF'

filegroup {

    name: "x",

    srcs: ["x.txt"],

    bazel_module: {bp2build_available: true},

  }

EOF



  run_soong bp2build

  if [[ ! -e "./out/soong/bp2build/pkg/BUILD.bazel" ]]; then

    fail "Expected pkg/BUILD.bazel to be generated"

  fi



  rm pkg/Android.bp



  run_soong bp2build

  if [[ -e "./out/soong/bp2build/pkg/BUILD.bazel" ]]; then

    fail "Expected pkg/BUILD.bazel to be deleted"

  fi

}



function test_bp2build_null_build_with_globs {

  setup