Merge "Remove bp2build and bazel from soong_ui" into main am: e51c6e4109 (b5943280) · Commits · e / os / android_build_soong

cmd/soong_ui/main.go

+4 −109

Original line number	Diff line number	Diff line
		@@ -18,7 +18,6 @@ import (
		"context"
		"flag"
		"fmt"
		"io/ioutil"
		"os"
		"path/filepath"
		"strconv"
		@@ -194,16 +193,13 @@ func main() {
		buildErrorFile := filepath.Join(logsDir, c.logsPrefix+"build_error")
		soongMetricsFile := filepath.Join(logsDir, c.logsPrefix+"soong_metrics")
		rbeMetricsFile := filepath.Join(logsDir, c.logsPrefix+"rbe_metrics.pb")
		bp2buildMetricsFile := filepath.Join(logsDir, c.logsPrefix+"bp2build_metrics.pb")
		soongBuildMetricsFile := filepath.Join(logsDir, c.logsPrefix+"soong_build_metrics.pb")

		metricsFiles := []string{
		buildErrorFile, // build error strings
		rbeMetricsFile, // high level metrics related to remote build execution.
		bp2buildMetricsFile, // high level metrics related to bp2build.
		soongMetricsFile, // high level metrics related to this build system.
		soongBuildMetricsFile, // high level metrics related to soong build(except bp2build)
		config.BazelMetricsDir(), // directory that contains a set of bazel metrics.
		soongBuildMetricsFile, // high level metrics related to soong build
		}

		os.MkdirAll(logsDir, 0777)
		@@ -293,38 +289,12 @@ func preProductConfigSetup(buildCtx build.Context, config build.Config) {
		}
		}

		removeBadTargetRename(buildCtx, config)

		// Create a source finder.
		f := build.NewSourceFinder(buildCtx, config)
		defer f.Shutdown()
		build.FindSources(buildCtx, config, f)
		}

		func removeBadTargetRename(ctx build.Context, config build.Config) {
		log := ctx.ContextImpl.Logger
		// find bad paths
		m, err := filepath.Glob(filepath.Join(config.OutDir(), "bazel", "output", "execroot", "__main__", "bazel-out", "mixed_builds_product-*", "bin", "tools", "metalava", "metalava"))
		if err != nil {
		log.Fatalf("Glob for invalid file failed %s", err)
		}
		for _, f := range m {
		info, err := os.Stat(f)
		if err != nil {
		log.Fatalf("Stat of invalid file %q failed %s", f, err)
		}
		// if it's a directory, leave it, but remove the files
		if !info.IsDir() {
		err = os.Remove(f)
		if err != nil {
		log.Fatalf("Remove of invalid file %q failed %s", f, err)
		} else {
		log.Verbosef("Removed %q", f)
		}
		}
		}
		}

		func dumpVar(ctx build.Context, config build.Config, args []string) {
		flags := flag.NewFlagSet("dumpvar", flag.ExitOnError)
		flags.SetOutput(ctx.Writer)
		@@ -607,81 +577,6 @@ func getCommand(args []string) (*command, []string, error) {
		return nil, nil, fmt.Errorf("Command not found: %q\nDid you mean one of these: %q", args[1], listFlags())
		}

		// For Bazel support, this moves files and directories from e.g. out/dist/$f to DIST_DIR/$f if necessary.
		func populateExternalDistDir(ctx build.Context, config build.Config) {
		// Make sure that internalDistDirPath and externalDistDirPath are both absolute paths, so we can compare them
		var err error
		var internalDistDirPath string
		var externalDistDirPath string
		if internalDistDirPath, err = filepath.Abs(config.DistDir()); err != nil {
		ctx.Fatalf("Unable to find absolute path of %s: %s", internalDistDirPath, err)
		}
		if externalDistDirPath, err = filepath.Abs(config.RealDistDir()); err != nil {
		ctx.Fatalf("Unable to find absolute path of %s: %s", externalDistDirPath, err)
		}
		if externalDistDirPath == internalDistDirPath {
		return
		}

		// Make sure the internal DIST_DIR actually exists before trying to read from it
		if _, err = os.Stat(internalDistDirPath); os.IsNotExist(err) {
		ctx.Println("Skipping Bazel dist dir migration - nothing to do!")
		return
		}

		// Make sure the external DIST_DIR actually exists before trying to write to it
		if err = os.MkdirAll(externalDistDirPath, 0755); err != nil {
		ctx.Fatalf("Unable to make directory %s: %s", externalDistDirPath, err)
		}

		ctx.Println("Populating external DIST_DIR...")

		populateExternalDistDirHelper(ctx, config, internalDistDirPath, externalDistDirPath)
		}

		func populateExternalDistDirHelper(ctx build.Context, config build.Config, internalDistDirPath string, externalDistDirPath string) {
		files, err := ioutil.ReadDir(internalDistDirPath)
		if err != nil {
		ctx.Fatalf("Can't read internal distdir %s: %s", internalDistDirPath, err)
		}
		for _, f := range files {
		internalFilePath := filepath.Join(internalDistDirPath, f.Name())
		externalFilePath := filepath.Join(externalDistDirPath, f.Name())

		if f.IsDir() {
		// Moving a directory - check if there is an existing directory to merge with
		externalLstat, err := os.Lstat(externalFilePath)
		if err != nil {
		if !os.IsNotExist(err) {
		ctx.Fatalf("Can't lstat external %s: %s", externalDistDirPath, err)
		}
		// Otherwise, if the error was os.IsNotExist, that's fine and we fall through to the rename at the bottom
		} else {
		if externalLstat.IsDir() {
		// Existing dir - try to merge the directories?
		populateExternalDistDirHelper(ctx, config, internalFilePath, externalFilePath)
		continue
		} else {
		// Existing file being replaced with a directory. Delete the existing file...
		if err := os.RemoveAll(externalFilePath); err != nil {
		ctx.Fatalf("Unable to remove existing %s: %s", externalFilePath, err)
		}
		}
		}
		} else {
		// Moving a file (not a dir) - delete any existing file or directory
		if err := os.RemoveAll(externalFilePath); err != nil {
		ctx.Fatalf("Unable to remove existing %s: %s", externalFilePath, err)
		}
		}

		// The actual move - do a rename instead of a copy in order to save disk space.
		if err := os.Rename(internalFilePath, externalFilePath); err != nil {
		ctx.Fatalf("Unable to rename %s -> %s due to error %s", internalFilePath, externalFilePath, err)
		}
		}
		}

		func setMaxFiles(ctx build.Context) {
		var limits syscall.Rlimit

ui/build/Android.bp

+0 −2

Original line number	Diff line number	Diff line
		@@ -46,7 +46,6 @@ bootstrap_go_package {
		"soong-ui-tracer",
		],
		srcs: [
		"bazel_metrics.go",
		"build.go",
		"cleanbuild.go",
		"config.go",
		@@ -75,7 +74,6 @@ bootstrap_go_package {
		"proc_sync_test.go",
		"rbe_test.go",
		"staging_snapshot_test.go",
		"upload_test.go",
		"util_test.go",
		],
		darwin: {

ui/build/bazel_metrics.go

deleted100644 → 0

+0 −136

Original line number	Diff line number	Diff line
		// Copyright 2023 Google Inc. All rights reserved.
		//
		// Licensed under the Apache License, Version 2.0 (the "License");
		// you may not use this file except in compliance with the License.
		// You may obtain a copy of the License at
		//
		// http://www.apache.org/licenses/LICENSE-2.0
		//
		// Unless required by applicable law or agreed to in writing, software
		// distributed under the License is distributed on an "AS IS" BASIS,
		// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
		// See the License for the specific language governing permissions and
		// limitations under the License.

		package build

		// This file contains functionality to parse bazel profile data into
		// a bazel_metrics proto, defined in build/soong/ui/metrics/bazel_metrics_proto
		// These metrics are later uploaded in upload.go

		import (
		"bufio"
		"os"
		"strconv"
		"strings"

		"android/soong/shared"
		"google.golang.org/protobuf/proto"

		bazel_metrics_proto "android/soong/ui/metrics/bazel_metrics_proto"
		)

		func parseTimingToNanos(str string) int64 {
		millisString := removeDecimalPoint(str)
		timingMillis, _ := strconv.ParseInt(millisString, 10, 64)
		return timingMillis * 1000000
		}

		func parsePercentageToTenThousandths(str string) int32 {
		percentageString := removeDecimalPoint(str)
		//remove the % at the end of the string
		percentage := strings.ReplaceAll(percentageString, "%", "")
		percentagePortion, _ := strconv.ParseInt(percentage, 10, 32)
		return int32(percentagePortion)
		}

		func removeDecimalPoint(numString string) string {
		// The format is always 0.425 or 10.425
		return strings.ReplaceAll(numString, ".", "")
		}

		func parseTotal(line string) int64 {
		words := strings.Fields(line)
		timing := words[3]
		return parseTimingToNanos(timing)
		}

		func parsePhaseTiming(line string) bazel_metrics_proto.PhaseTiming {
		words := strings.Fields(line)
		getPhaseNameAndTimingAndPercentage := func([]string) (string, int64, int32) {
		// Sample lines include:
		// Total launch phase time 0.011 s 2.59%
		// Total target pattern evaluation phase time 0.011 s 2.59%
		var beginning int
		var end int
		for ind, word := range words {
		if word == "Total" {
		beginning = ind + 1
		} else if beginning > 0 && word == "phase" {
		end = ind
		break
		}
		}
		phaseName := strings.Join(words[beginning:end], " ")

		// end is now "phase" - advance by 2 for timing and 4 for percentage
		percentageString := words[end+4]
		timingString := words[end+2]
		timing := parseTimingToNanos(timingString)
		percentagePortion := parsePercentageToTenThousandths(percentageString)
		return phaseName, timing, percentagePortion
		}

		phaseName, timing, portion := getPhaseNameAndTimingAndPercentage(words)
		phaseTiming := bazel_metrics_proto.PhaseTiming{}
		phaseTiming.DurationNanos = &timing
		phaseTiming.PortionOfBuildTime = &portion

		phaseTiming.PhaseName = &phaseName
		return phaseTiming
		}

		// This method takes a file created by bazel's --analyze-profile mode and
		// writes bazel metrics data to the provided filepath.
		func ProcessBazelMetrics(bazelProfileFile string, bazelMetricsFile string, ctx Context, config Config) {
		if bazelProfileFile == "" {
		return
		}

		readBazelProto := func(filepath string) bazel_metrics_proto.BazelMetrics {
		//serialize the proto, write it
		bazelMetrics := bazel_metrics_proto.BazelMetrics{}

		file, err := os.ReadFile(filepath)
		if err != nil {
		ctx.Fatalln("Error reading metrics file\n", err)
		}

		scanner := bufio.NewScanner(strings.NewReader(string(file)))
		scanner.Split(bufio.ScanLines)

		var phaseTimings []*bazel_metrics_proto.PhaseTiming
		for scanner.Scan() {
		line := scanner.Text()
		if strings.HasPrefix(line, "Total run time") {
		total := parseTotal(line)
		bazelMetrics.Total = &total
		} else if strings.HasPrefix(line, "Total") {
		phaseTiming := parsePhaseTiming(line)
		phaseTimings = append(phaseTimings, &phaseTiming)
		}
		}
		bazelMetrics.PhaseTimings = phaseTimings
		bazelMetrics.BesId = proto.String(config.besId)

		return bazelMetrics
		}

		if _, err := os.Stat(bazelProfileFile); err != nil {
		// We can assume bazel didn't run if the profile doesn't exist
		return
		}
		bazelProto := readBazelProto(bazelProfileFile)
		bazelProto.ExitCode = proto.Int32(config.bazelExitCode)
		shared.Save(&bazelProto, bazelMetricsFile)
		}

ui/build/build.go

+0 −18

Original line number	Diff line number	Diff line
		@@ -135,22 +135,6 @@ const (
		RunBuildTests = 1 << iota
		)

		// checkBazelMode fails the build if there are conflicting arguments for which bazel
		// build mode to use.
		func checkBazelMode(ctx Context, config Config) {
		count := 0
		if config.bazelProdMode {
		count++
		}
		if config.bazelStagingMode {
		count++
		}
		if count > 1 {
		ctx.Fatalln("Conflicting bazel mode.\n" +
		"Do not specify more than one of --bazel-mode and --bazel-mode-staging ")
		}
		}

		// checkProblematicFiles fails the build if existing Android.mk or CleanSpec.mk files are found at the root of the tree.
		func checkProblematicFiles(ctx Context) {
		files := []string{"Android.mk", "CleanSpec.mk"}
		@@ -262,8 +246,6 @@ func Build(ctx Context, config Config) {

		defer waitForDist(ctx)

		checkBazelMode(ctx, config)

		// checkProblematicFiles aborts the build if Android.mk or CleanSpec.mk are found at the root of the tree.
		checkProblematicFiles(ctx)

ui/build/config.go

+8 −88

Original line number	Diff line number	Diff line
		@@ -72,7 +72,6 @@ type configImpl struct {
		checkbuild bool
		dist bool
		jsonModuleGraph bool
		bp2build bool
		queryview bool
		reportMkMetrics bool // Collect and report mk2bp migration progress metrics.
		soongDocs bool
		@@ -88,8 +87,6 @@ type configImpl struct {
		buildStartedTime int64 // For metrics-upload-only - manually specify a build-started time
		buildFromSourceStub bool
		ensureAllowlistIntegrity bool // For CI builds - make sure modules are mixed-built
		bazelExitCode int32 // For b runs - necessary for updating NonZeroExit
		besId string // For b runs, to identify the BuildEventService logs

		// From the product config
		katiArgs []string
		@@ -108,16 +105,11 @@ type configImpl struct {

		pathReplaced bool

		bazelProdMode bool
		bazelStagingMode bool

		// Set by multiproduct_kati
		emptyNinjaFile bool

		metricsUploader string

		bazelForceEnabledModules string

		includeTags []string
		sourceRootDirs []string

		@@ -450,11 +442,6 @@ func NewConfig(ctx Context, args ...string) Config {
		}
		}

		bpd := ret.BazelMetricsDir()
		if err := os.RemoveAll(bpd); err != nil {
		ctx.Fatalf("Unable to remove bazel profile directory %q: %v", bpd, err)
		}

		c := Config{ret}
		storeConfigMetrics(ctx, c)
		return c
		@@ -538,8 +525,6 @@ func buildConfig(config Config) *smpb.BuildConfig {
		ForceUseGoma: proto.Bool(config.ForceUseGoma()),
		UseGoma: proto.Bool(config.UseGoma()),
		UseRbe: proto.Bool(config.UseRBE()),
		BazelMixedBuild: proto.Bool(config.BazelBuildEnabled()),
		ForceDisableBazelMixedBuild: proto.Bool(config.IsBazelMixedBuildForceDisabled()),
		NinjaWeightListSource: getNinjaWeightListSourceInMetric(config.NinjaWeightListSource()),
		}
		c.Targets = append(c.Targets, config.arguments...)
		@@ -794,10 +779,6 @@ func (c *configImpl) parseArgs(ctx Context, args []string) {
		c.reportMkMetrics = true
		} else if arg == "--multitree-build" {
		c.multitreeBuild = true
		} else if arg == "--bazel-mode" {
		c.bazelProdMode = true
		} else if arg == "--bazel-mode-staging" {
		c.bazelStagingMode = true
		} else if arg == "--search-api-dir" {
		c.searchApiDir = true
		} else if strings.HasPrefix(arg, "--ninja_weight_source=") {
		@@ -832,8 +813,6 @@ func (c *configImpl) parseArgs(ctx Context, args []string) {
		buildCmd = strings.TrimPrefix(buildCmd, "\"")
		buildCmd = strings.TrimSuffix(buildCmd, "\"")
		ctx.Metrics.SetBuildCommand([]string{buildCmd})
		} else if strings.HasPrefix(arg, "--bazel-force-enabled-modules=") {
		c.bazelForceEnabledModules = strings.TrimPrefix(arg, "--bazel-force-enabled-modules=")
		} else if strings.HasPrefix(arg, "--build-started-time-unix-millis=") {
		buildTimeStr := strings.TrimPrefix(arg, "--build-started-time-unix-millis=")
		val, err := strconv.ParseInt(buildTimeStr, 10, 64)
		@@ -878,8 +857,6 @@ func (c *configImpl) parseArgs(ctx Context, args []string) {
		c.dist = true
		} else if arg == "json-module-graph" {
		c.jsonModuleGraph = true
		} else if arg == "bp2build" {
		c.bp2build = true
		} else if arg == "queryview" {
		c.queryview = true
		} else if arg == "soong_docs" {
		@@ -976,13 +953,12 @@ func (c *configImpl) SoongBuildInvocationNeeded() bool {
		return true
		}

		if !c.JsonModuleGraph() && !c.Bp2Build() && !c.Queryview() && !c.SoongDocs() {
		if !c.JsonModuleGraph() && !c.Queryview() && !c.SoongDocs() {
		// Command line was empty, the default Ninja target is built
		return true
		}

		// bp2build + dist may be used to dist bp2build logs but does not require SoongBuildInvocation
		if c.Dist() && !c.Bp2Build() {
		if c.Dist() {
		return true
		}

		@@ -1012,14 +988,6 @@ func (c *configImpl) NinjaArgs() []string {
		return c.ninjaArgs
		}

		func (c *configImpl) BazelOutDir() string {
		return filepath.Join(c.OutDir(), "bazel")
		}

		func (c *configImpl) bazelOutputBase() string {
		return filepath.Join(c.BazelOutDir(), "output")
		}

		func (c *configImpl) SoongOutDir() string {
		return filepath.Join(c.OutDir(), "soong")
		}
		@@ -1058,14 +1026,6 @@ func (c *configImpl) UsedEnvFile(tag string) string {
		return shared.JoinPath(c.SoongOutDir(), usedEnvFile+"."+tag)
		}

		func (c *configImpl) Bp2BuildFilesMarkerFile() string {
		return shared.JoinPath(c.SoongOutDir(), "bp2build_files_marker")
		}

		func (c *configImpl) Bp2BuildWorkspaceMarkerFile() string {
		return shared.JoinPath(c.SoongOutDir(), "bp2build_workspace_marker")
		}

		func (c *configImpl) SoongDocsHtml() string {
		return shared.JoinPath(c.SoongOutDir(), "docs/soong_build.html")
		}
		@@ -1111,10 +1071,6 @@ func (c *configImpl) JsonModuleGraph() bool {
		return c.jsonModuleGraph
		}

		func (c *configImpl) Bp2Build() bool {
		return c.bp2build
		}

		func (c *configImpl) Queryview() bool {
		return c.queryview
		}
		@@ -1306,7 +1262,7 @@ func (c *configImpl) canSupportRBE() bool {

		func (c *configImpl) UseRBE() bool {
		// These alternate modes of running Soong do not use RBE / reclient.
		if c.Bp2Build() \|\| c.Queryview() \|\| c.JsonModuleGraph() {
		if c.Queryview() \|\| c.JsonModuleGraph() {
		return false
		}

		@@ -1323,10 +1279,6 @@ func (c *configImpl) UseRBE() bool {
		return false
		}

		func (c *configImpl) BazelBuildEnabled() bool {
		return c.bazelProdMode \|\| c.bazelStagingMode
		}

		func (c *configImpl) StartRBE() bool {
		if !c.UseRBE() {
		return false
		@@ -1679,12 +1631,6 @@ func (c *configImpl) LogsDir() string {
		return absDir
		}

		// BazelMetricsDir returns the <logs dir>/bazel_metrics directory
		// where the bazel profiles are located.
		func (c *configImpl) BazelMetricsDir() string {
		return filepath.Join(c.LogsDir(), "bazel_metrics")
		}

		// MkFileMetrics returns the file path for make-related metrics.
		func (c *configImpl) MkMetrics() string {
		return filepath.Join(c.LogsDir(), "mk_metrics.pb")
		@@ -1698,28 +1644,6 @@ func (c *configImpl) EmptyNinjaFile() bool {
		return c.emptyNinjaFile
		}

		func (c *configImpl) IsBazelMixedBuildForceDisabled() bool {
		return c.Environment().IsEnvTrue("BUILD_BROKEN_DISABLE_BAZEL")
		}

		func (c *configImpl) IsPersistentBazelEnabled() bool {
		return c.Environment().IsEnvTrue("USE_PERSISTENT_BAZEL")
		}

		// GetBazeliskBazelVersion returns the Bazel version to use for this build,
		// or the empty string if the current canonical prod Bazel should be used.
		// This environment variable should only be set to debug the build system.
		// The Bazel version, if set, will be passed to Bazelisk, and Bazelisk will
		// handle downloading and invoking the correct Bazel binary.
		func (c *configImpl) GetBazeliskBazelVersion() string {
		value, _ := c.Environment().Get("USE_BAZEL_VERSION")
		return value
		}

		func (c *configImpl) BazelModulesForceEnabledByFlag() string {
		return c.bazelForceEnabledModules
		}

		func (c *configImpl) SkipMetricsUpload() bool {
		return c.skipMetricsUpload
		}
		@@ -1737,10 +1661,6 @@ func (c *configImpl) BuildStartedTimeOrDefault(defaultTime time.Time) time.Time
		return time.UnixMilli(c.buildStartedTime)
		}

		func (c *configImpl) BazelExitCode() int32 {
		return c.bazelExitCode
		}

		func GetMetricsUploader(topDir string, env *Environment) string {
		if p, ok := env.Get("METRICS_UPLOADER"); ok {
		metricsUploader := filepath.Join(topDir, p)