Merge "Update symlinks in output directory when TOP dir changes" into main (a9022509) · Commits · e / os / android_build_soong

tests/lib.sh

+13 −0

Original line number	Diff line number	Diff line
		@@ -8,6 +8,11 @@ HARDWIRED_MOCK_TOP=

		REAL_TOP="$(readlink -f "$(dirname "$0")"/../../..)"

		function make_mock_top {
		mock=$(mktemp -t -d st.XXXXX)
		echo "$mock"
		}

		if [[ -n "$HARDWIRED_MOCK_TOP" ]]; then
		MOCK_TOP="$HARDWIRED_MOCK_TOP"
		else
		@@ -198,3 +203,11 @@ function scan_and_run_tests {
		info "Completed test case \e[96;1m$f\e[0m"
		done
		}

		function move_mock_top {
		MOCK_TOP2=$(make_mock_top)
		rm -rf $MOCK_TOP2
		mv $MOCK_TOP $MOCK_TOP2
		MOCK_TOP=$MOCK_TOP2
		trap cleanup_mock_top EXIT
		}

tests/run_integration_tests.sh

+1 −1

Original line number	Diff line number	Diff line
		@@ -23,4 +23,4 @@ TEST_BAZEL=true extra_build_params=--bazel-mode-staging "$TOP/build/soong/tests/
		"$TOP/build/soong/tests/apex_cc_module_arch_variant_tests.sh" "aosp_cf_arm64_phone" "armv8-a" "cortex-a53"

		"$TOP/build/bazel/ci/b_test.sh"

		"$TOP/build/soong/tests/symlinks_path_test.sh"

tests/symlinks_path_test.sh

0 → 100755

+51 −0

Original line number	Diff line number	Diff line
		#!/bin/bash -eu

		set -o pipefail

		# Test that relative symlinks work by recreating the bug in b/259191764
		# In some cases, developers prefer to move their checkouts. This causes
		# issues in that symlinked files (namely, the bazel wrapper script)
		# cannot be found. As such, we implemented relative symlinks so that a
		# moved checkout doesn't need a full clean before rebuilding.
		# The bazel output base will still need to be removed, as Starlark
		# doesn't seem to support relative symlinks yet.

		source "$(dirname "$0")/lib.sh"

		function check_link_has_mock_top_prefix {
		input_link=$1
		link_target=`readlink $input_link`
		if [[ $link_target != "$MOCK_TOP"* ]]; then
		echo "Symlink for file $input_link -> $link_target doesn't start with $MOCK_TOP"
		exit 1
		fi
		}

		function test_symlinks_updated_when_top_dir_changed {
		setup

		mkdir -p a
		touch a/g.txt
		cat > a/Android.bp <<'EOF'
		filegroup {
		name: "g",
		srcs: ["g.txt"],
		bazel_module: {bp2build_available: true},
		}
		EOF
		# A directory under $MOCK_TOP
		outdir=out2

		# Modify OUT_DIR in a subshell so it doesn't affect the top level one.
		(export OUT_DIR=$MOCK_TOP/$outdir; run_soong bp2build && run_bazel build --config=bp2build --config=ci //a:g)

		g_txt="out2/soong/workspace/a/g.txt"
		check_link_has_mock_top_prefix "$g_txt"

		move_mock_top

		(export OUT_DIR=$MOCK_TOP/$outdir; run_soong bp2build && run_bazel build --config=bp2build --config=ci //a:g)
		check_link_has_mock_top_prefix "$g_txt"
		}

		scan_and_run_tests
		No newline at end of file

ui/build/soong.go

+118 −0

Original line number	Diff line number	Diff line
		@@ -16,10 +16,13 @@ package build

		import (
		"fmt"
		"io/fs"
		"os"
		"path/filepath"
		"strconv"
		"strings"
		"sync"
		"sync/atomic"

		"android/soong/bazel"
		"android/soong/ui/metrics"
		@@ -50,6 +53,13 @@ const (
		bootstrapEpoch = 1
		)

		var (
		// Used during parallel update of symlinks in out directory to reflect new
		// TOP dir.
		symlinkWg sync.WaitGroup
		numFound, numUpdated uint32
		)

		func writeEnvironmentFile(_ Context, envFile string, envDeps map[string]string) error {
		data, err := shared.EnvFileContents(envDeps)
		if err != nil {
		@@ -465,10 +475,118 @@ func checkEnvironmentFile(ctx Context, currentEnv *Environment, envFile string)
		}
		}

		func updateSymlinks(ctx Context, dir, prevCWD, cwd string) error {
		defer symlinkWg.Done()

		visit := func(path string, d fs.DirEntry, err error) error {
		if d.IsDir() && path != dir {
		symlinkWg.Add(1)
		go updateSymlinks(ctx, path, prevCWD, cwd)
		return filepath.SkipDir
		}
		f, err := d.Info()
		if err != nil {
		return err
		}
		// If the file is not a symlink, we don't have to update it.
		if f.Mode()&os.ModeSymlink != os.ModeSymlink {
		return nil
		}

		atomic.AddUint32(&numFound, 1)
		target, err := os.Readlink(path)
		if err != nil {
		return err
		}
		if strings.HasPrefix(target, prevCWD) &&
		(len(target) == len(prevCWD) \|\| target[len(prevCWD)] == '/') {
		target = filepath.Join(cwd, target[len(prevCWD):])
		if err := os.Remove(path); err != nil {
		return err
		}
		if err := os.Symlink(target, path); err != nil {
		return err
		}
		atomic.AddUint32(&numUpdated, 1)
		}
		return nil
		}

		if err := filepath.WalkDir(dir, visit); err != nil {
		return err
		}
		return nil
		}

		func fixOutDirSymlinks(ctx Context, config Config, outDir string) error {
		cwd, err := os.Getwd()
		if err != nil {
		return err
		}

		// Record the .top as the very last thing in the function.
		tf := filepath.Join(outDir, ".top")
		defer func() {
		if err := os.WriteFile(tf, []byte(cwd), 0644); err != nil {
		fmt.Fprintf(os.Stderr, fmt.Sprintf("Unable to log CWD: %v", err))
		}
		}()

		// Find the previous working directory if it was recorded.
		var prevCWD string
		pcwd, err := os.ReadFile(tf)
		if err != nil {
		if os.IsNotExist(err) {
		// No previous working directory recorded, nothing to do.
		return nil
		}
		return err
		}
		prevCWD = strings.Trim(string(pcwd), "\n")

		if prevCWD == cwd {
		// We are in the same source dir, nothing to update.
		return nil
		}

		symlinkWg.Add(1)
		if err := updateSymlinks(ctx, outDir, prevCWD, cwd); err != nil {
		return err
		}
		symlinkWg.Wait()
		ctx.Println(fmt.Sprintf("Updated %d/%d symlinks in dir %v", numUpdated, numFound, outDir))
		return nil
		}

		func migrateOutputSymlinks(ctx Context, config Config) error {
		// Figure out the real out directory ("out" could be a symlink).
		outDir := config.OutDir()
		s, err := os.Lstat(outDir)
		if err != nil {
		if os.IsNotExist(err) {
		// No out dir exists, no symlinks to migrate.
		return nil
		}
		return err
		}
		if s.Mode()&os.ModeSymlink == os.ModeSymlink {
		target, err := filepath.EvalSymlinks(outDir)
		if err != nil {
		return err
		}
		outDir = target
		}
		return fixOutDirSymlinks(ctx, config, outDir)
		}

		func runSoong(ctx Context, config Config) {
		ctx.BeginTrace(metrics.RunSoong, "soong")
		defer ctx.EndTrace()

		if err := migrateOutputSymlinks(ctx, config); err != nil {
		ctx.Fatalf("failed to migrate output directory to current TOP dir: %v", err)
		}

		// We have two environment files: .available is the one with every variable,
		// .used with the ones that were actually used. The latter is used to
		// determine whether Soong needs to be re-run since why re-run it if only