Revert "Modify symlink_forest to rerun when soong_build has changed." (aa5cc2cd) · Commits · e / os / android_build_soong

bp2build/symlink_forest.go

+40 −39

Original line number	Original line	Diff line number	Diff line
	@@ -22,6 +22,7 @@ import (
	"regexp"		"regexp"
	"sort"		"sort"
	"strconv"		"strconv"
			"strings"
	"sync"		"sync"
	"sync/atomic"		"sync/atomic"

	@@ -31,12 +32,19 @@ import (
	)		)

	// A tree structure that describes what to do at each directory in the created		// A tree structure that describes what to do at each directory in the created
	// symlink tree. Currently, it is used to enumerate which files/directories		// symlink tree. Currently it is used to enumerate which files/directories
	// should be excluded from symlinking. Each instance of "node" represents a file		// should be excluded from symlinking. Each instance of "node" represents a file
	// or a directory. If excluded is true, then that file/directory should be		// or a directory. If excluded is true, then that file/directory should be
	// excluded from symlinking. Otherwise, the node is not excluded, but one of its		// excluded from symlinking. Otherwise, the node is not excluded, but one of its
	// descendants is (otherwise the node in question would not exist)		// descendants is (otherwise the node in question would not exist)

			// This is a version int written to a file called symlink_forest_version at the root of the
			// symlink forest. If the version here does not match the version in the file, then we'll
			// clean the whole symlink forest and recreate it. This number can be bumped whenever there's
			// an incompatible change to the forest layout or a bug in incrementality that needs to be fixed
			// on machines that may still have the bug present in their forest.
			const symlinkForestVersion = 2

	type instructionsNode struct {		type instructionsNode struct {
	name string		name string
	excluded bool // If false, this is just an intermediate node		excluded bool // If false, this is just an intermediate node
	@@ -185,7 +193,7 @@ func symlinkIntoForest(topdir, dst, src string) uint64 {
	srcPath := shared.JoinPath(topdir, src)		srcPath := shared.JoinPath(topdir, src)
	dstPath := shared.JoinPath(topdir, dst)		dstPath := shared.JoinPath(topdir, dst)

	// Check whether a symlink already exists.		// Check if a symlink already exists.
	if dstInfo, err := os.Lstat(dstPath); err != nil {		if dstInfo, err := os.Lstat(dstPath); err != nil {
	if !os.IsNotExist(err) {		if !os.IsNotExist(err) {
	fmt.Fprintf(os.Stderr, "Failed to lstat '%s': %s", dst, err)		fmt.Fprintf(os.Stderr, "Failed to lstat '%s': %s", dst, err)
	@@ -232,51 +240,46 @@ func isDir(path string, fi os.FileInfo) bool {
	return false		return false
	}		}

	// Returns the hash of the soong_build binary to determine whether we should		// maybeCleanSymlinkForest will remove the whole symlink forest directory if the version recorded
	// force symlink_forest to re-execute		// in the symlink_forest_version file is not equal to symlinkForestVersion.
	// This is similar to a version number increment - but that shouldn't be required		func maybeCleanSymlinkForest(topdir, forest string, verbose bool) error {
	// for every update to this file		versionFilePath := shared.JoinPath(topdir, forest, "symlink_forest_version")
	func getSoongBuildMTime() int64 {		versionFileContents, err := os.ReadFile(versionFilePath)
	binaryPath, err := os.Executable()		if err != nil && !os.IsNotExist(err) {
	if err != nil {		return err
	fmt.Fprintf(os.Stderr, "error finding executable path %s\n", err)
	os.Exit(1)
	}		}
			versionFileString := strings.TrimSpace(string(versionFileContents))
	info, err := os.Stat(binaryPath)		symlinkForestVersionString := strconv.Itoa(symlinkForestVersion)
			if err != nil \|\| versionFileString != symlinkForestVersionString {
			if verbose {
			fmt.Fprintf(os.Stderr, "Old symlink_forest_version was %q, current is %q. Cleaning symlink forest before recreating...\n", versionFileString, symlinkForestVersionString)
			}
			err = os.RemoveAll(shared.JoinPath(topdir, forest))
	if err != nil {		if err != nil {
	fmt.Fprintf(os.Stderr, "error stating executable path %s\n", err)		return err
	}		}
			}
	return info.ModTime().UnixMilli()		return nil
	}		}

	// maybeCleanSymlinkForest will remove the whole symlink forest directory if the soong_build		// maybeWriteVersionFile will write the symlink_forest_version file containing symlinkForestVersion
	// binary has changed since the last execution.		// if it doesn't exist already. If it exists we know it must contain symlinkForestVersion because
	func maybeCleanSymlinkForest(topdir, forest string, verbose bool, soongBuildMTime int64) error {		// we checked for that already in maybeCleanSymlinkForest
	mtimeFilePath := shared.JoinPath(topdir, forest, "soong_build_mtime")		func maybeWriteVersionFile(topdir, forest string) error {
	mtimeFileContents, err := os.ReadFile(mtimeFilePath)		versionFilePath := shared.JoinPath(topdir, forest, "symlink_forest_version")
	if err != nil && !os.IsNotExist(err) {		_, err := os.Stat(versionFilePath)
			if err != nil {
			if !os.IsNotExist(err) {
	return err		return err
	}		}
			err = os.WriteFile(versionFilePath, []byte(strconv.Itoa(symlinkForestVersion)+"\n"), 0666)
	if string(soongBuildMTime) != string(mtimeFileContents) {
	err = os.RemoveAll(shared.JoinPath(topdir, forest))
	if err != nil {		if err != nil {
	return err		return err
	}		}
	}		}

	return nil		return nil
	}		}

	func writeSoongBuildMTimeFile(topdir, forest string, mtime int64) error {
	hashFilePath := shared.JoinPath(topdir, forest, "soong_build_mtime")
	contents := []byte(strconv.FormatInt(mtime, 10))

	return os.WriteFile(hashFilePath, contents, 0666)
	}

	// Recursively plants a symlink forest at forestDir. The symlink tree will		// Recursively plants a symlink forest at forestDir. The symlink tree will
	// contain every file in buildFilesDir and srcDir excluding the files in		// contain every file in buildFilesDir and srcDir excluding the files in
	// instructions. Collects every directory encountered during the traversal of		// instructions. Collects every directory encountered during the traversal of
	@@ -470,10 +473,7 @@ func PlantSymlinkForest(verbose bool, topdir string, forest string, buildFiles s
	symlinkCount: atomic.Uint64{},		symlinkCount: atomic.Uint64{},
	}		}

	// Check whether soong_build has been modified since the last run		err := maybeCleanSymlinkForest(topdir, forest, verbose)
	soongBuildMTime := getSoongBuildMTime()

	err := maybeCleanSymlinkForest(topdir, forest, verbose, soongBuildMTime)
	if err != nil {		if err != nil {
	fmt.Fprintln(os.Stderr, err)		fmt.Fprintln(os.Stderr, err)
	os.Exit(1)		os.Exit(1)
	@@ -491,10 +491,11 @@ func PlantSymlinkForest(verbose bool, topdir string, forest string, buildFiles s
	deps = append(deps, dep)		deps = append(deps, dep)
	}		}

	err = writeSoongBuildMTimeFile(topdir, forest, soongBuildMTime)		err = maybeWriteVersionFile(topdir, forest)
	if err != nil {		if err != nil {
	fmt.Fprintln(os.Stderr, err)		fmt.Fprintln(os.Stderr, err)
	os.Exit(1)		os.Exit(1)
	}		}

	return deps, context.mkdirCount.Load(), context.symlinkCount.Load()		return deps, context.mkdirCount.Load(), context.symlinkCount.Load()
	}		}

tests/run_integration_tests.sh

+0 −1

Original line number	Original line	Diff line number	Diff line
	@@ -10,7 +10,6 @@ TOP="$(readlink -f "$(dirname "$0")"/../../..)"
	"$TOP/build/soong/tests/persistent_bazel_test.sh"		"$TOP/build/soong/tests/persistent_bazel_test.sh"
	"$TOP/build/soong/tests/soong_test.sh"		"$TOP/build/soong/tests/soong_test.sh"
	"$TOP/build/soong/tests/stale_metrics_files_test.sh"		"$TOP/build/soong/tests/stale_metrics_files_test.sh"
	"$TOP/build/soong/tests/symlink_forest_rerun_test.sh"
	"$TOP/prebuilts/build-tools/linux-x86/bin/py3-cmd" "$TOP/build/bazel/ci/rbc_dashboard.py" aosp_arm64-userdebug		"$TOP/prebuilts/build-tools/linux-x86/bin/py3-cmd" "$TOP/build/bazel/ci/rbc_dashboard.py" aosp_arm64-userdebug

	# The following tests build against the full source tree and don't rely on the		# The following tests build against the full source tree and don't rely on the

tests/symlink_forest_rerun_test.sh

deleted100755 → 0

+0 −42

Original line number	Original line	Diff line number	Diff line
	#!/bin/bash -eu

	set -o pipefail

	# Tests that symlink_Forest will rerun if soong_build has schanged

	source "$(dirname "$0")/lib.sh"

	function test_symlink_forest_reruns {
	setup

	mkdir -p a
	touch a/g.txt
	cat > a/Android.bp <<'EOF'
	filegroup {
	name: "g",
	srcs: ["g.txt"],
	}
	EOF

	run_soong g

	mtime=`cat out/soong/workspace/soong_build_mtime`
	# rerun with no changes - ensure that it hasn't changed
	run_soong g
	newmtime=`cat out/soong/workspace/soong_build_mtime`
	if [[ ! "$mtime" == "$mtime" ]]; then
	fail "symlink forest reran when it shouldn't have"
	fi

	# change exit codes to force a soong_build rebuild.
	sed -i 's/os.Exit(1)/os.Exit(2)/g' build/soong/bp2build/symlink_forest.go

	run_soong g
	newmtime=`cat out/soong/workspace/soong_build_mtime`
	if [[ "$mtime" == "$newmtime" ]]; then
	fail "symlink forest did not rerun when it should have"
	fi

	}

	scan_and_run_tests