Merge "Make globs compatible with hash-based ninja semantics" into main

package main

import (

	"bytes"

	"encoding/json"

	"errors"

	"flag"

	"android/soong/android/allowlists"

	"android/soong/bp2build"

	"android/soong/shared"

	"github.com/google/blueprint"

	"github.com/google/blueprint/bootstrap"

	"github.com/google/blueprint/deptools"

	"github.com/google/blueprint/metrics"

	"github.com/google/blueprint/pathtools"

	"github.com/google/blueprint/proptools"

	androidProtobuf "google.golang.org/protobuf/android"

)

	availableEnvFile string

	usedEnvFile      string

	globFile    string

	globListDir string

	delveListen string

	delvePath   string

	flag.StringVar(&cmdlineArgs.SoongOutDir, "soong_out", "", "Soong output directory (usually $TOP/out/soong)")

	flag.StringVar(&availableEnvFile, "available_env", "", "File containing available environment variables")

	flag.StringVar(&usedEnvFile, "used_env", "", "File containing used environment variables")

	flag.StringVar(&globFile, "globFile", "build-globs.ninja", "the Ninja file of globs to output")

	flag.StringVar(&globListDir, "globListDir", "", "the directory containing the glob list files")

	flag.StringVar(&cmdlineArgs.OutDir, "out", "", "the ninja builddir directory")

	flag.StringVar(&cmdlineArgs.ModuleListFile, "l", "", "file that lists filepaths to parse")

	ctx.Context.PrintJSONGraphAndActions(graphFile, actionsFile)

}

func writeBuildGlobsNinjaFile(ctx *android.Context) {

	ctx.EventHandler.Begin("globs_ninja_file")

	defer ctx.EventHandler.End("globs_ninja_file")

	globDir := bootstrap.GlobDirectory(ctx.Config().SoongOutDir(), globListDir)

	err := bootstrap.WriteBuildGlobsNinjaFile(&bootstrap.GlobSingleton{

		GlobLister: ctx.Globs,

		GlobFile:   globFile,

		GlobDir:    globDir,

		SrcDir:     ctx.SrcDir(),

	}, ctx.Config())

	maybeQuit(err, "")

}

func writeDepFile(outputFile string, eventHandler *metrics.EventHandler, ninjaDeps []string) {

	eventHandler.Begin("ninja_deps")

	defer eventHandler.End("ninja_deps")

}

// runSoongOnlyBuild runs the standard Soong build in a number of different modes.

func runSoongOnlyBuild(ctx *android.Context, extraNinjaDeps []string) string {

// It returns the path to the output file (usually the ninja file) and the deps that need

// to trigger a soong rerun.

func runSoongOnlyBuild(ctx *android.Context) (string, []string) {

	ctx.EventHandler.Begin("soong_build")

	defer ctx.EventHandler.End("soong_build")

	ninjaDeps, err := bootstrap.RunBlueprint(cmdlineArgs.Args, stopBefore, ctx.Context, ctx.Config())

	maybeQuit(err, "")

	ninjaDeps = append(ninjaDeps, extraNinjaDeps...)

	writeBuildGlobsNinjaFile(ctx)

	// Convert the Soong module graph into Bazel BUILD files.

	switch ctx.Config().BuildMode {

	case android.GenerateQueryView:

		queryviewMarkerFile := cmdlineArgs.BazelQueryViewDir + ".marker"

		runQueryView(cmdlineArgs.BazelQueryViewDir, queryviewMarkerFile, ctx)

		writeDepFile(queryviewMarkerFile, ctx.EventHandler, ninjaDeps)

		return queryviewMarkerFile

		return queryviewMarkerFile, ninjaDeps

	case android.GenerateModuleGraph:

		writeJsonModuleGraphAndActions(ctx, cmdlineArgs)

		writeDepFile(cmdlineArgs.ModuleGraphFile, ctx.EventHandler, ninjaDeps)

		return cmdlineArgs.ModuleGraphFile

		return cmdlineArgs.ModuleGraphFile, ninjaDeps

	case android.GenerateDocFile:

		// TODO: we could make writeDocs() return the list of documentation files

		// written and add them to the .d file. Then soong_docs would be re-run

		// whenever one is deleted.

		err := writeDocs(ctx, shared.JoinPath(topDir, cmdlineArgs.DocFile))

		maybeQuit(err, "error building Soong documentation")

		writeDepFile(cmdlineArgs.DocFile, ctx.EventHandler, ninjaDeps)

		return cmdlineArgs.DocFile

		return cmdlineArgs.DocFile, ninjaDeps

	default:

		// The actual output (build.ninja) was written in the RunBlueprint() call

		// above

		writeDepFile(cmdlineArgs.OutFile, ctx.EventHandler, ninjaDeps)

		if needToWriteNinjaHint(ctx) {

			writeNinjaHint(ctx)

		}

		return cmdlineArgs.OutFile

		return cmdlineArgs.OutFile, ninjaDeps

	}

}

func main() {

	flag.Parse()

	soongStartTime := time.Now()

	shared.ReexecWithDelveMaybe(delveListen, delvePath)

	android.InitSandbox(topDir)

		configuration.SetAllowMissingDependencies()

	}

	extraNinjaDeps := []string{configuration.ProductVariablesFileName, usedEnvFile}

	if shared.IsDebugging() {

		// Add a non-existent file to the dependencies so that soong_build will rerun when the debugger is

		// enabled even if it completed successfully.

		extraNinjaDeps = append(extraNinjaDeps, filepath.Join(configuration.SoongOutDir(), "always_rerun_for_delve"))

	}

	// Bypass configuration.Getenv, as LOG_DIR does not need to be dependency tracked. By definition, it will

	// change between every CI build, so tracking it would require re-running Soong for every build.

	metricsDir := availableEnv["LOG_DIR"]

	ctx.SetIncrementalAnalysis(incremental)

	ctx.Register()

	finalOutputFile := runSoongOnlyBuild(ctx, extraNinjaDeps)

	finalOutputFile, ninjaDeps := runSoongOnlyBuild(ctx)

	ninjaDeps = append(ninjaDeps, usedEnvFile)

	if shared.IsDebugging() {

		// Add a non-existent file to the dependencies so that soong_build will rerun when the debugger is

		// enabled even if it completed successfully.

		ninjaDeps = append(ninjaDeps, filepath.Join(configuration.SoongOutDir(), "always_rerun_for_delve"))

	}

	writeDepFile(finalOutputFile, ctx.EventHandler, ninjaDeps)

	if ctx.GetIncrementalEnabled() {

		data, err := shared.EnvFileContents(configuration.EnvDeps())

	writeUsedEnvironmentFile(configuration)

	err = writeGlobFile(ctx.EventHandler, finalOutputFile, ctx.Globs(), soongStartTime)

	maybeQuit(err, "")

	// Touch the output file so that it's the newest file created by soong_build.

	// This is necessary because, if soong_build generated any files which

	// are ninja inputs to the main output file, then ninja would superfluously

	data, err := shared.EnvFileContents(configuration.EnvDeps())

	maybeQuit(err, "error writing used environment file '%s'\n", usedEnvFile)

	if preexistingData, err := os.ReadFile(path); err != nil {

		if !os.IsNotExist(err) {

			maybeQuit(err, "error reading used environment file '%s'", usedEnvFile)

	err = pathtools.WriteFileIfChanged(path, data, 0666)

	maybeQuit(err, "error writing used environment file '%s'", usedEnvFile)

}

	} else if bytes.Equal(preexistingData, data) {

		// used environment file is unchanged

		return

func writeGlobFile(eventHandler *metrics.EventHandler, finalOutFile string, globs pathtools.MultipleGlobResults, soongStartTime time.Time) error {

	eventHandler.Begin("writeGlobFile")

	defer eventHandler.End("writeGlobFile")

	globsFile, err := os.Create(shared.JoinPath(topDir, finalOutFile+".globs"))

	if err != nil {

		return err

	}

	err = os.WriteFile(path, data, 0666)

	maybeQuit(err, "error writing used environment file '%s'", usedEnvFile)

	defer globsFile.Close()

	globsFileEncoder := json.NewEncoder(globsFile)

	for _, glob := range globs {

		if err := globsFileEncoder.Encode(glob); err != nil {

			return err

		}

	}

	return os.WriteFile(

		shared.JoinPath(topDir, finalOutFile+".globs_time"),

		[]byte(fmt.Sprintf("%d\n", soongStartTime.UnixMicro())),

		0666,

	)

}

func touch(path string) {

  run_soong

  local -r ninja_mtime1=$(stat -c "%y" out/soong/build."${target_product}".ninja)

  local glob_deps_file=out/soong/globs/"${target_product}"/0.d

  run_soong

  local -r ninja_mtime2=$(stat -c "%y" out/soong/build."${target_product}".ninja)

  # There is an ineffiencency in glob that requires bpglob to rerun once for each glob to update

  # the entry in the .ninja_log.  It doesn't update the output file, but we can detect the rerun

  # by checking if the deps file was created.

  if [ ! -e "$glob_deps_file" ]; then

    fail "Glob deps file missing after second build"

  fi

  local -r glob_deps_mtime2=$(stat -c "%y" "$glob_deps_file")

  if [[ "$ninja_mtime1" != "$ninja_mtime2" ]]; then

    fail "Ninja file rewritten on null incremental build"

  fi

  run_soong

  local -r ninja_mtime3=$(stat -c "%y" out/soong/build."${target_product}".ninja)

  local -r glob_deps_mtime3=$(stat -c "%y" "$glob_deps_file")

  if [[ "$ninja_mtime2" != "$ninja_mtime3" ]]; then

    fail "Ninja file rewritten on null incremental build"

  fi

  # The bpglob commands should not rerun after the first incremental build.

  if [[ "$glob_deps_mtime2" != "$glob_deps_mtime3" ]]; then

    fail "Glob deps file rewritten on second null incremental build"

  fi

}

function test_add_file_to_glob() {

	}

}

func (c *configImpl) NamedGlobFile(name string) string {

	return shared.JoinPath(c.SoongOutDir(), "globs-"+name+".ninja")

}

func (c *configImpl) UsedEnvFile(tag string) string {

	if v, ok := c.environ.Get("TARGET_PRODUCT"); ok {

		return shared.JoinPath(c.SoongOutDir(), usedEnvFile+"."+v+c.CoverageSuffix()+"."+tag)

package build

import (

	"encoding/json"

	"errors"

	"fmt"

	"io/fs"

	"os"

	"path/filepath"

	"runtime"

	"slices"

	"strconv"

	"strings"

	"sync"

	// bootstrapEpoch is used to determine if an incremental build is incompatible with the current

	// version of bootstrap and needs cleaning before continuing the build.  Increment this for

	// incompatible changes, for example when moving the location of the bpglob binary that is

	// incompatible changes, for example when moving the location of a microfactory binary that is

	// executed during bootstrap before the primary builder has had a chance to update the path.

	bootstrapEpoch = 1

)

	var allArgs []string

	allArgs = append(allArgs, pb.specificArgs...)

	globPathName := getGlobPathNameFromPrimaryBuilderFactory(config, pb)

	allArgs = append(allArgs,

		"--globListDir", globPathName,

		"--globFile", pb.config.NamedGlobFile(globPathName))

	allArgs = append(allArgs, commonArgs...)

	allArgs = append(allArgs, environmentArgs(pb.config, pb.name)...)

	}

	allArgs = append(allArgs, "Android.bp")

	globfiles := bootstrap.GlobFileListFiles(bootstrap.GlobDirectory(config.SoongOutDir(), globPathName))

	return bootstrap.PrimaryBuilderInvocation{

		Implicits:   globfiles,

		Implicits:   []string{pb.output + ".glob_results"},

		Outputs:     []string{pb.output},

		Args:        allArgs,

		Description: pb.description,

				os.Remove(file)

			}

		}

		for _, globFile := range bootstrapGlobFileList(config) {

			os.Remove(globFile)

		}

		os.Remove(soongNinjaFile + ".globs")

		os.Remove(soongNinjaFile + ".globs_time")

		os.Remove(soongNinjaFile + ".glob_results")

		// Mark the tree as up to date with the current epoch by writing the epoch marker file.

		writeEmptyFile(ctx, epochPath)

	}

}

func bootstrapGlobFileList(config Config) []string {

	return []string{

		config.NamedGlobFile(getGlobPathName(config)),

		config.NamedGlobFile(jsonModuleGraphTag),

		config.NamedGlobFile(queryviewTag),

		config.NamedGlobFile(soongDocsTag),

	}

}

func bootstrapBlueprint(ctx Context, config Config) {

	ctx.BeginTrace(metrics.RunSoong, "blueprint bootstrap")

	defer ctx.EndTrace()

		runGoTests:  !config.skipSoongTests,

		// If we want to debug soong_build, we need to compile it for debugging

		debugCompilation:          delvePort != "",

		subninjas:                 bootstrapGlobFileList(config),

		primaryBuilderInvocations: invocations,

	}

	// The glob ninja files are generated during the main build phase. However, the

	// primary buildifer invocation depends on all of its glob files, even before

	// it's been run. Generate a "empty" glob ninja file on the first run,

	// so that the files can be there to satisfy the dependency.

	for _, pb := range pbfs {

		globPathName := getGlobPathNameFromPrimaryBuilderFactory(config, pb)

		globNinjaFile := config.NamedGlobFile(globPathName)

		if _, err := os.Stat(globNinjaFile); os.IsNotExist(err) {

			err := bootstrap.WriteBuildGlobsNinjaFile(&bootstrap.GlobSingleton{

				GlobLister: func() pathtools.MultipleGlobResults { return nil },

				GlobFile:   globNinjaFile,

				GlobDir:    bootstrap.GlobDirectory(config.SoongOutDir(), globPathName),

				SrcDir:     ".",

			}, blueprintConfig)

			if err != nil {

				ctx.Fatal(err)

			}

		} else if err != nil {

			ctx.Fatal(err)

		}

	}

	// since `bootstrap.ninja` is regenerated unconditionally, we ignore the deps, i.e. little

	// reason to write a `bootstrap.ninja.d` file

	_, err := bootstrap.RunBlueprint(blueprintArgs, bootstrap.DoEverything, blueprintCtx, blueprintConfig)

		}

	}()

	runMicrofactory(ctx, config, "bpglob", "github.com/google/blueprint/bootstrap/bpglob",

		map[string]string{"github.com/google/blueprint": "build/blueprint"})

	ninja := func(targets ...string) {

		ctx.BeginTrace(metrics.RunSoong, "bootstrap")

		defer ctx.EndTrace()

		targets = append(targets, config.SoongNinjaFile())

	}

	for _, target := range targets {

		if err := checkGlobs(ctx, target); err != nil {

			ctx.Fatalf("Error checking globs: %s", err.Error())

		}

	}

	beforeSoongTimestamp := time.Now()

	ninja(targets...)

	}

}

// checkGlobs manages the globs that cause soong to rerun.

//

// When soong_build runs, it will run globs. It will write all the globs

// it ran into the "{finalOutFile}.globs" file. Then every build,

// soong_ui will check that file, rerun the globs, and if they changed

// from the results that soong_build got, update the ".glob_results"

// file, causing soong_build to rerun. The ".glob_results" file will

// be empty on the first run of soong_build, because we don't know

// what the globs are yet, but also remain empty until the globs change

// so that we don't run soong_build a second time unnecessarily.

// Both soong_build and soong_ui will also update a ".globs_time" file

// with the time that they ran at every build. When soong_ui checks

// globs, it only reruns globs whose dependencies are newer than the

// time in the ".globs_time" file.

func checkGlobs(ctx Context, finalOutFile string) error {

	ctx.BeginTrace(metrics.RunSoong, "check_globs")

	defer ctx.EndTrace()

	st := ctx.Status.StartTool()

	st.Status("Running globs...")

	defer st.Finish()

	globsFile, err := os.Open(finalOutFile + ".globs")

	if errors.Is(err, fs.ErrNotExist) {

		// if the glob file doesn't exist, make sure the glob_results file exists and is empty.

		if err := os.MkdirAll(filepath.Dir(finalOutFile), 0777); err != nil {

			return err

		}

		f, err := os.Create(finalOutFile + ".glob_results")

		if err != nil {

			return err

		}

		return f.Close()

	} else if err != nil {

		return err

	}

	defer globsFile.Close()

	globsFileDecoder := json.NewDecoder(globsFile)

	globsTimeBytes, err := os.ReadFile(finalOutFile + ".globs_time")

	if err != nil {

		return err

	}

	globsTimeMicros, err := strconv.ParseInt(strings.TrimSpace(string(globsTimeBytes)), 10, 64)

	if err != nil {

		return err

	}

	globCheckStartTime := time.Now().UnixMicro()

	globsChan := make(chan pathtools.GlobResult)

	errorsChan := make(chan error)

	wg := sync.WaitGroup{}

	hasChangedGlobs := false

	for i := 0; i < runtime.NumCPU()*2; i++ {

		wg.Add(1)

		go func() {

			for cachedGlob := range globsChan {

				// If we've already determined we have changed globs, just finish consuming

				// the channel without doing any more checks.

				if hasChangedGlobs {

					continue

				}

				// First, check if any of the deps are newer than the last time globs were checked.

				// If not, we don't need to rerun the glob.

				hasNewDep := false

				for _, dep := range cachedGlob.Deps {

					info, err := os.Stat(dep)

					if err != nil {

						errorsChan <- err

						continue

					}

					if info.ModTime().UnixMicro() > globsTimeMicros {

						hasNewDep = true

						break

					}

				}

				if !hasNewDep {

					continue

				}

				// Then rerun the glob and check if we got the same result as before.

				result, err := pathtools.Glob(cachedGlob.Pattern, cachedGlob.Excludes, pathtools.FollowSymlinks)

				if err != nil {

					errorsChan <- err

				} else {

					if !slices.Equal(result.Matches, cachedGlob.Matches) {

						hasChangedGlobs = true

					}

				}

			}

			wg.Done()

		}()

	}

	go func() {

		wg.Wait()

		close(errorsChan)

	}()

	errorsWg := sync.WaitGroup{}

	errorsWg.Add(1)

	var errFromGoRoutines error

	go func() {

		for result := range errorsChan {

			if errFromGoRoutines == nil {

				errFromGoRoutines = result

			}

		}

		errorsWg.Done()

	}()

	var cachedGlob pathtools.GlobResult

	for globsFileDecoder.More() {

		if err := globsFileDecoder.Decode(&cachedGlob); err != nil {

			return err

		}

		// Need to clone the GlobResult because the json decoder will

		// reuse the same slice allocations.

		globsChan <- cachedGlob.Clone()

	}

	close(globsChan)

	errorsWg.Wait()

	if errFromGoRoutines != nil {

		return errFromGoRoutines

	}

	// Update the globs_time file whether or not we found changed globs,

	// so that we don't rerun globs in the future that we just saw didn't change.

	err = os.WriteFile(

		finalOutFile+".globs_time",

		[]byte(fmt.Sprintf("%d\n", globCheckStartTime)),

		0666,

	)

	if err != nil {

		return err

	}

	if hasChangedGlobs {

		fmt.Fprintf(os.Stdout, "Globs changed, rerunning soong...\n")

		// Write the current time to the glob_results file. We just need

		// some unique value to trigger a rerun, it doesn't matter what it is.

		err = os.WriteFile(

			finalOutFile+".glob_results",

			[]byte(fmt.Sprintf("%d\n", globCheckStartTime)),

			0666,

		)

		if err != nil {

			return err

		}

	}

	return nil

}

// loadSoongBuildMetrics reads out/soong_build_metrics.pb if it was generated by soong_build and copies the

// events stored in it into the soong_ui trace to provide introspection into how long the different phases of

// soong_build are taking.

	// out/build_date.txt is considered a "source file"

	buildDatetimeFilePath := filepath.Join(outDir, "build_date.txt")

	// bpglob is built explicitly using Microfactory

	bpglob := filepath.Join(config.SoongOutDir(), "bpglob")

	// release-config files are generated from the initial lunch or Kati phase

	// before running soong and ninja.

	releaseConfigDir := filepath.Join(outDir, "soong", "release-config")

			line == extraVariablesFilePath ||

			line == dexpreoptConfigFilePath ||

			line == buildDatetimeFilePath ||

			line == bpglob ||

			strings.HasPrefix(line, releaseConfigDir) ||

			buildFingerPrintFilePattern.MatchString(line) {

			// Leaf node is in one of Soong's bootstrap directories, which do not have