Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 0cf44d5e authored by Jeongik Cha's avatar Jeongik Cha
Browse files

Add --ninja_weight_source option 

It has 3 options for now.
* ninja_log: uses ninja log file for data source
* evenly_distributed: pass empty list for ninja to consider every work as the same
* not_used: do not use critical path logic in ninja

In addition, I added the option in the metrics to track

Bug: 271527305
Test: m --ninja_weight_source=ninja_log|empty|not_used
Change-Id: Ib4c812c20606a34b17d3f0edb71057b477c4f90e
parent 767ce714

ui/build/config.go

+40 −0
Original line number Diff line number Diff line
@@ -118,8 +118,21 @@ type configImpl struct { 
	bazelForceEnabledModules string



	includeTags []string



	// Data source to write ninja weight list

	ninjaWeightListSource NinjaWeightListSource

}



type NinjaWeightListSource uint



const (

	// ninja doesn't use weight list.

	NOT_USED NinjaWeightListSource = iota

	// ninja uses weight list based on previous builds by ninja log

	NINJA_LOG

	// ninja thinks every task has the same weight.

	EVENLY_DISTRIBUTED

)

const srcDirFileCheck = "build/soong/root.bp"



var buildFiles = []string{"Android.mk", "Android.bp"}
@@ -528,6 +541,17 @@ func storeConfigMetrics(ctx Context, config Config) { 
	ctx.Metrics.SystemResourceInfo(s)

}



func getNinjaWeightListSourceInMetric(s NinjaWeightListSource) *smpb.NinjaWeightListSource {

	switch s {

	case NINJA_LOG:

		return smpb.NinjaWeightListSource_NINJA_LOG.Enum()

	case EVENLY_DISTRIBUTED:

		return smpb.NinjaWeightListSource_EVENLY_DISTRIBUTED.Enum()

	default:

		return smpb.NinjaWeightListSource_NOT_USED.Enum()

	}

}



func buildConfig(config Config) *smpb.BuildConfig {

	c := &smpb.BuildConfig{

		ForceUseGoma:                proto.Bool(config.ForceUseGoma()),
@@ -535,6 +559,7 @@ func buildConfig(config Config) *smpb.BuildConfig { 
		UseRbe:                      proto.Bool(config.UseRBE()),

		BazelMixedBuild:             proto.Bool(config.BazelBuildEnabled()),

		ForceDisableBazelMixedBuild: proto.Bool(config.IsBazelMixedBuildForceDisabled()),

		NinjaWeightListSource:       getNinjaWeightListSourceInMetric(config.NinjaWeightListSource()),

	}

	c.Targets = append(c.Targets, config.arguments...)
@@ -797,6 +822,17 @@ func (c *configImpl) parseArgs(ctx Context, args []string) { 
			c.bazelStagingMode = true

		} else if arg == "--search-api-dir" {

			c.searchApiDir = true

		} else if strings.HasPrefix(arg, "--ninja_weight_source=") {

			source := strings.TrimPrefix(arg, "--ninja_weight_source=")

			if source == "ninja_log" {

				c.ninjaWeightListSource = NINJA_LOG

			} else if source == "evenly_distributed" {

				c.ninjaWeightListSource = EVENLY_DISTRIBUTED

			} else if source == "not_used" {

				c.ninjaWeightListSource = NOT_USED

			} else {

				ctx.Fatalf("unknown option for ninja_weight_source: %s", source)

			}

		} else if strings.HasPrefix(arg, "--build-command=") {

			buildCmd := strings.TrimPrefix(arg, "--build-command=")

			// remove quotations
@@ -1091,6 +1127,10 @@ func (c *configImpl) MultitreeBuild() bool { 
	return c.multitreeBuild

}



func (c *configImpl) NinjaWeightListSource() NinjaWeightListSource {

	return c.ninjaWeightListSource

}



func (c *configImpl) SkipKati() bool {

	return c.skipKati

}

ui/build/config_test.go

+11 −0
Original line number Diff line number Diff line
@@ -1032,6 +1032,7 @@ func TestBuildConfig(t *testing.T) { 
				UseRbe:                      proto.Bool(false),

				BazelMixedBuild:             proto.Bool(false),

				ForceDisableBazelMixedBuild: proto.Bool(false),

				NinjaWeightListSource:       smpb.NinjaWeightListSource_NOT_USED.Enum(),

			},

		},

		{
@@ -1043,6 +1044,7 @@ func TestBuildConfig(t *testing.T) { 
				UseRbe:                      proto.Bool(false),

				BazelMixedBuild:             proto.Bool(false),

				ForceDisableBazelMixedBuild: proto.Bool(false),

				NinjaWeightListSource:       smpb.NinjaWeightListSource_NOT_USED.Enum(),

			},

		},

		{
@@ -1054,6 +1056,7 @@ func TestBuildConfig(t *testing.T) { 
				UseRbe:                      proto.Bool(false),

				BazelMixedBuild:             proto.Bool(false),

				ForceDisableBazelMixedBuild: proto.Bool(false),

				NinjaWeightListSource:       smpb.NinjaWeightListSource_NOT_USED.Enum(),

			},

		},

		{
@@ -1065,6 +1068,7 @@ func TestBuildConfig(t *testing.T) { 
				UseRbe:                      proto.Bool(true),

				BazelMixedBuild:             proto.Bool(false),

				ForceDisableBazelMixedBuild: proto.Bool(false),

				NinjaWeightListSource:       smpb.NinjaWeightListSource_NOT_USED.Enum(),

			},

		},

		{
@@ -1076,6 +1080,7 @@ func TestBuildConfig(t *testing.T) { 
				UseRbe:                      proto.Bool(false),

				BazelMixedBuild:             proto.Bool(false),

				ForceDisableBazelMixedBuild: proto.Bool(true),

				NinjaWeightListSource:       smpb.NinjaWeightListSource_NOT_USED.Enum(),

			},

		},

		{
@@ -1088,6 +1093,7 @@ func TestBuildConfig(t *testing.T) { 
				UseRbe:                      proto.Bool(false),

				BazelMixedBuild:             proto.Bool(false),

				ForceDisableBazelMixedBuild: proto.Bool(false),

				NinjaWeightListSource:       smpb.NinjaWeightListSource_NOT_USED.Enum(),

			},

		},

		{
@@ -1100,6 +1106,7 @@ func TestBuildConfig(t *testing.T) { 
				UseRbe:                      proto.Bool(false),

				BazelMixedBuild:             proto.Bool(true),

				ForceDisableBazelMixedBuild: proto.Bool(false),

				NinjaWeightListSource:       smpb.NinjaWeightListSource_NOT_USED.Enum(),

			},

		},

		{
@@ -1112,6 +1119,7 @@ func TestBuildConfig(t *testing.T) { 
				UseRbe:                      proto.Bool(false),

				BazelMixedBuild:             proto.Bool(true),

				ForceDisableBazelMixedBuild: proto.Bool(false),

				NinjaWeightListSource:       smpb.NinjaWeightListSource_NOT_USED.Enum(),

			},

		},

		{
@@ -1124,6 +1132,7 @@ func TestBuildConfig(t *testing.T) { 
				UseRbe:                      proto.Bool(false),

				BazelMixedBuild:             proto.Bool(true),

				ForceDisableBazelMixedBuild: proto.Bool(false),

				NinjaWeightListSource:       smpb.NinjaWeightListSource_NOT_USED.Enum(),

			},

		},

		{
@@ -1138,6 +1147,7 @@ func TestBuildConfig(t *testing.T) { 
				BazelMixedBuild:             proto.Bool(false),

				Targets:                     []string{"droid", "dist"},

				ForceDisableBazelMixedBuild: proto.Bool(false),

				NinjaWeightListSource:       smpb.NinjaWeightListSource_NOT_USED.Enum(),

			},

		},

		{
@@ -1156,6 +1166,7 @@ func TestBuildConfig(t *testing.T) { 
				UseRbe:                      proto.Bool(true),

				BazelMixedBuild:             proto.Bool(true),

				ForceDisableBazelMixedBuild: proto.Bool(true),

				NinjaWeightListSource:       smpb.NinjaWeightListSource_NOT_USED.Enum(),

			},

		},

	}

ui/build/ninja.go

+48 −0
Original line number Diff line number Diff line
@@ -33,6 +33,46 @@ const ( 
	ninjaEnvFileName = "ninja.environment"

)



func useNinjaBuildLog(ctx Context, config Config, cmd *Cmd) {

	ninjaLogFile := filepath.Join(config.OutDir(), ".ninja_log")

	data, err := os.ReadFile(ninjaLogFile)

	var outputBuilder strings.Builder

	if err == nil {

		lines := strings.Split(strings.TrimSpace(string(data)), "\n")

		// ninja log: <start>	<end>	<restat>	<name>	<cmdhash>

		// ninja weight list: <name>,<end-start+1>

		for _, line := range lines {

			if strings.HasPrefix(line, "#") {

				continue

			}

			fields := strings.Split(line, "\t")

			path := fields[3]

			start, err := strconv.Atoi(fields[0])

			if err != nil {

				continue

			}

			end, err := strconv.Atoi(fields[1])

			if err != nil {

				continue

			}

			outputBuilder.WriteString(path)

			outputBuilder.WriteString(",")

			outputBuilder.WriteString(strconv.Itoa(end-start+1) + "\n")

		}

	}

	// If there is no ninja log file, just pass empty ninja weight list.

	// Because it is still efficient with critical path calculation logic even without weight.



	weightListFile := filepath.Join(config.OutDir(), ".ninja_weight_list")



	err = os.WriteFile(weightListFile, []byte(outputBuilder.String()), 0644)

	if err == nil {

		cmd.Args = append(cmd.Args, "-o", "usesweightlist="+weightListFile)

	} else {

		ctx.Panicf("Could not write ninja weight list file %s", err)

	}

}



// Constructs and runs the Ninja command line with a restricted set of

// environment variables. It's important to restrict the environment Ninja runs

// for hermeticity reasons, and to avoid spurious rebuilds.
@@ -85,6 +125,14 @@ func runNinjaForBuild(ctx Context, config Config) { 
		cmd.Environment.AppendFromKati(config.KatiEnvFile())

	}



	switch config.NinjaWeightListSource() {

	case NINJA_LOG:

		useNinjaBuildLog(ctx, config, cmd)

	case EVENLY_DISTRIBUTED:

		// pass empty weight list means ninja considers every tasks's weight as 1(default value).

		cmd.Args = append(cmd.Args, "-o", "usesweightlist=/dev/null")

	}



	// Allow both NINJA_ARGS and NINJA_EXTRA_ARGS, since both have been

	// used in the past to specify extra ninja arguments.

	if extra, ok := cmd.Environment.Get("NINJA_ARGS"); ok {

ui/metrics/metrics_proto/metrics.pb.go

+299 −210

File changed.

Preview size limit exceeded, changes collapsed.

ui/metrics/metrics_proto/metrics.proto

+11 −0
Original line number Diff line number Diff line
@@ -153,6 +153,11 @@ message BuildConfig { 


  // Whether the user explicitly disabled bazel mixed builds for this build.

  optional bool force_disable_bazel_mixed_build = 7;



  // NOT_USED - ninja doesn't use weight list.

  // NINJA_LOG - ninja uses weight list based on previous builds by ninja log

  // EVENLY_DISTRIBUTED - ninja thinks every task has the same weight.

  optional NinjaWeightListSource ninja_weight_list_source = 8 [default = NOT_USED];

}



message SystemResourceInfo {
@@ -337,3 +342,9 @@ message JobInfo { 
  // Description of a job

  optional string job_description = 2;

}



enum NinjaWeightListSource {

  NOT_USED = 0;

  NINJA_LOG = 1;

  EVENLY_DISTRIBUTED = 2;

}
 
 No newline at end of file