Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit ca390b2f authored by Kousik Kumar's avatar Kousik Kumar
Browse files

Update symlinks in output directory when TOP dir changes 

When TOP dir changes and we reuse the same output between the old and new checkouts, we need to rewrite all the symlinks used by Bazel and other tools in the out-directory to point to the new top directory. Otherwise, if the old source dir is deleted, the build will fail.

I used the OUT_DIR/soong/soong.environment.available file to find out the previous PWD.

Tested:
1. Create source dir 1, run build, create source dir 2, remote source dir 1, reuse out dir and rerun build => build succeeded with this change.
2. m libc after moving build TOP. Only the analysis phase was rerun, the actual build was not rerun.

Bug: b/300498226

Change-Id: I196625baa1f4efe7a4734accfa1f0be7c98a7920
parent 72928074

tests/lib.sh

+13 −0
Original line number Diff line number Diff line
@@ -8,6 +8,11 @@ HARDWIRED_MOCK_TOP= 


REAL_TOP="$(readlink -f "$(dirname "$0")"/../../..)"



function make_mock_top {

  mock=$(mktemp -t -d st.XXXXX)

  echo "$mock"

}



if [[ -n "$HARDWIRED_MOCK_TOP" ]]; then

  MOCK_TOP="$HARDWIRED_MOCK_TOP"

else
@@ -197,3 +202,11 @@ function scan_and_run_tests { 
    info "Completed test case \e[96;1m$f\e[0m"

  done

}



function move_mock_top {

  MOCK_TOP2=$(make_mock_top)

  rm -rf $MOCK_TOP2

  mv $MOCK_TOP $MOCK_TOP2

  MOCK_TOP=$MOCK_TOP2

  trap cleanup_mock_top EXIT

}

tests/run_integration_tests.sh

+1 −1
Original line number Diff line number Diff line
@@ -23,4 +23,4 @@ TEST_BAZEL=true extra_build_params=--bazel-mode-staging "$TOP/build/soong/tests/ 
"$TOP/build/soong/tests/apex_cc_module_arch_variant_tests.sh" "aosp_cf_arm64_phone" "armv8-a" "cortex-a53"



"$TOP/build/bazel/ci/b_test.sh"

"$TOP/build/soong/tests/symlinks_path_test.sh"

tests/symlinks_path_test.sh

0 → 100755
+51 −0
Original line number Diff line number Diff line 
#!/bin/bash -eu



set -o pipefail



# Test that relative symlinks work by recreating the bug in b/259191764

# In some cases, developers prefer to move their checkouts. This causes

# issues in that symlinked files (namely, the bazel wrapper script)

# cannot be found. As such, we implemented relative symlinks so that a

# moved checkout doesn't need a full clean before rebuilding.

# The bazel output base will still need to be removed, as Starlark

# doesn't seem to support relative symlinks yet.



source "$(dirname "$0")/lib.sh"



function check_link_has_mock_top_prefix {

  input_link=$1

  link_target=`readlink $input_link`

  if [[ $link_target != "$MOCK_TOP"* ]]; then

    echo "Symlink for file $input_link -> $link_target doesn't start with $MOCK_TOP"

    exit 1

  fi

}



function test_symlinks_updated_when_top_dir_changed {

  setup



  mkdir -p a

  touch a/g.txt

  cat > a/Android.bp <<'EOF'

filegroup {

    name: "g",

    srcs: ["g.txt"],

    bazel_module: {bp2build_available: true},

}

EOF

  # A directory under $MOCK_TOP

  outdir=out2



  # Modify OUT_DIR in a subshell so it doesn't affect the top level one.

  (export OUT_DIR=$MOCK_TOP/$outdir; run_soong bp2build && run_bazel build --config=bp2build --config=ci //a:g)



  g_txt="out2/soong/workspace/a/g.txt"

  check_link_has_mock_top_prefix "$g_txt"



  move_mock_top



  (export OUT_DIR=$MOCK_TOP/$outdir; run_soong bp2build && run_bazel build --config=bp2build --config=ci //a:g)

  check_link_has_mock_top_prefix "$g_txt"

}



scan_and_run_tests
 
 No newline at end of file

ui/build/soong.go

+118 −0
Original line number Diff line number Diff line
@@ -16,10 +16,13 @@ package build 


import (

	"fmt"

	"io/fs"

	"os"

	"path/filepath"

	"strconv"

	"strings"

	"sync"

	"sync/atomic"



	"android/soong/bazel"

	"android/soong/ui/metrics"
@@ -50,6 +53,13 @@ const ( 
	bootstrapEpoch = 1

)



var (

	// Used during parallel update of symlinks in out directory to reflect new

	// TOP dir.

	symlinkWg            sync.WaitGroup

	numFound, numUpdated uint32

)



func writeEnvironmentFile(_ Context, envFile string, envDeps map[string]string) error {

	data, err := shared.EnvFileContents(envDeps)

	if err != nil {
@@ -465,10 +475,118 @@ func checkEnvironmentFile(ctx Context, currentEnv *Environment, envFile string) 
	}

}



func updateSymlinks(ctx Context, dir, prevCWD, cwd string) error {

	defer symlinkWg.Done()



	visit := func(path string, d fs.DirEntry, err error) error {

		if d.IsDir() && path != dir {

			symlinkWg.Add(1)

			go updateSymlinks(ctx, path, prevCWD, cwd)

			return filepath.SkipDir

		}

		f, err := d.Info()

		if err != nil {

			return err

		}

		// If the file is not a symlink, we don't have to update it.

		if f.Mode()&os.ModeSymlink != os.ModeSymlink {

			return nil

		}



		atomic.AddUint32(&numFound, 1)

		target, err := os.Readlink(path)

		if err != nil {

			return err

		}

		if strings.HasPrefix(target, prevCWD) &&

			(len(target) == len(prevCWD) || target[len(prevCWD)] == '/') {

			target = filepath.Join(cwd, target[len(prevCWD):])

			if err := os.Remove(path); err != nil {

				return err

			}

			if err := os.Symlink(target, path); err != nil {

				return err

			}

			atomic.AddUint32(&numUpdated, 1)

		}

		return nil

	}



	if err := filepath.WalkDir(dir, visit); err != nil {

		return err

	}

	return nil

}



func fixOutDirSymlinks(ctx Context, config Config, outDir string) error {

	cwd, err := os.Getwd()

	if err != nil {

		return err

	}



	// Record the .top as the very last thing in the function.

	tf := filepath.Join(outDir, ".top")

	defer func() {

		if err := os.WriteFile(tf, []byte(cwd), 0644); err != nil {

			fmt.Fprintf(os.Stderr, fmt.Sprintf("Unable to log CWD: %v", err))

		}

	}()



	// Find the previous working directory if it was recorded.

	var prevCWD string

	pcwd, err := os.ReadFile(tf)

	if err != nil {

		if os.IsNotExist(err) {

			// No previous working directory recorded, nothing to do.

			return nil

		}

		return err

	}

	prevCWD = strings.Trim(string(pcwd), "\n")



	if prevCWD == cwd {

		// We are in the same source dir, nothing to update.

		return nil

	}



	symlinkWg.Add(1)

	if err := updateSymlinks(ctx, outDir, prevCWD, cwd); err != nil {

		return err

	}

	symlinkWg.Wait()

	ctx.Println(fmt.Sprintf("Updated %d/%d symlinks in dir %v", numUpdated, numFound, outDir))

	return nil

}



func migrateOutputSymlinks(ctx Context, config Config) error {

	// Figure out the real out directory ("out" could be a symlink).

	outDir := config.OutDir()

	s, err := os.Lstat(outDir)

	if err != nil {

		if os.IsNotExist(err) {

			// No out dir exists, no symlinks to migrate.

			return nil

		}

		return err

	}

	if s.Mode()&os.ModeSymlink == os.ModeSymlink {

		target, err := filepath.EvalSymlinks(outDir)

		if err != nil {

			return err

		}

		outDir = target

	}

	return fixOutDirSymlinks(ctx, config, outDir)

}



func runSoong(ctx Context, config Config) {

	ctx.BeginTrace(metrics.RunSoong, "soong")

	defer ctx.EndTrace()



	if err := migrateOutputSymlinks(ctx, config); err != nil {

		ctx.Fatalf("failed to migrate output directory to current TOP dir: %v", err)

	}



	// We have two environment files: .available is the one with every variable,

	// .used with the ones that were actually used. The latter is used to

	// determine whether Soong needs to be re-run since why re-run it if only