Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 8b8bec3b authored by Colin Cross's avatar Colin Cross
Browse files

Create a highmem pool and put metalava into it 

Create a highmem pool based on the total RAM and the number of CPUs,
with an override via the NINJA_HIGHMEM_NUM_JOBS variable.  Put
metalava into the highmem pool.

Ninja does not support nested pools, and when goma or RBE is enabled
the maximum ninja parallelism is set very high with local jobs in a
local pool.  When both the local pool and highmem pool are enabled,
the total number of local jobs will be as high as the sum of the sizes
of the two pools.  Keep the highmem pool limited to 1/16th of the
local pool when remote builds are enabled to try to minimize the
effect while still limiting highmem jobs.

Fixes: 142644983
Test: m nothing, examine pools
Test: m USE_GOMA=true nothing, examine pools
Change-Id: Id79f11f44948992960ac34ecf831dacbe21bd332
parent b1d8c99e

android/config.go

+4 −0
Original line number Diff line number Diff line
@@ -827,6 +827,10 @@ func (c *config) UseRBE() bool { 
	return Bool(c.productVariables.UseRBE)

}



func (c *config) UseRemoteBuild() bool {

	return c.UseGoma() || c.UseRBE()

}



func (c *config) RunErrorProne() bool {

	return c.IsEnvTrue("RUN_ERROR_PRONE")

}

android/defs.go

+3 −0
Original line number Diff line number Diff line
@@ -99,6 +99,9 @@ var ( 


	// Used only when USE_GOMA=true is set, to restrict non-goma jobs to the local parallelism value

	localPool = blueprint.NewBuiltinPool("local_pool")



	// Used for processes that need significant RAM to ensure there are not too many running in parallel.

	highmemPool = blueprint.NewBuiltinPool("highmem_pool")

)



func init() {

android/module.go

+1 −1
Original line number Diff line number Diff line
@@ -1338,7 +1338,7 @@ func (m *moduleContext) Variable(pctx PackageContext, name, value string) { 
func (m *moduleContext) Rule(pctx PackageContext, name string, params blueprint.RuleParams,

	argNames ...string) blueprint.Rule {



	if (m.config.UseGoma() || m.config.UseRBE()) && params.Pool == nil {

	if m.config.UseRemoteBuild() && params.Pool == nil {

		// When USE_GOMA=true or USE_RBE=true are set and the rule is not supported by goma/RBE, restrict

		// jobs to the local parallelism value

		params.Pool = localPool

android/package_ctx.go

+1 −1
Original line number Diff line number Diff line
@@ -109,7 +109,7 @@ func (p PackageContext) RuleFunc(name string, 
		if len(ctx.errors) > 0 {

			return params, ctx.errors[0]

		}

		if (ctx.Config().UseGoma() || ctx.Config().UseRBE()) && params.Pool == nil {

		if ctx.Config().UseRemoteBuild() && params.Pool == nil {

			// When USE_GOMA=true or USE_RBE=true are set and the rule is not supported by

			// goma/RBE, restrict jobs to the local parallelism value

			params.Pool = localPool

android/rule_builder.go

+27 −0
Original line number Diff line number Diff line
@@ -33,6 +33,8 @@ type RuleBuilder struct { 
	temporariesSet map[WritablePath]bool

	restat         bool

	sbox           bool

	highmem        bool

	remoteable     RemoteRuleSupports

	sboxOutDir     WritablePath

	missingDeps    []string

}
@@ -87,6 +89,19 @@ func (r *RuleBuilder) Restat() *RuleBuilder { 
	return r

}



// HighMem marks the rule as a high memory rule, which will limit how many run in parallel with other high memory

// rules.

func (r *RuleBuilder) HighMem() *RuleBuilder {

	r.highmem = true

	return r

}



// Remoteable marks the rule as supporting remote execution.

func (r *RuleBuilder) Remoteable(supports RemoteRuleSupports) *RuleBuilder {

	r.remoteable = supports

	return r

}



// Sbox marks the rule as needing to be wrapped by sbox. The WritablePath should point to the output

// directory that sbox will wipe. It should not be written to by any other rule. sbox will ensure

// that all outputs have been written, and will discard any output files that were not specified.
@@ -401,6 +416,17 @@ func (r *RuleBuilder) Build(pctx PackageContext, ctx BuilderContext, name string 
		rspFileContent = "$in"

	}



	var pool blueprint.Pool

	if ctx.Config().UseGoma() && r.remoteable&SUPPORTS_GOMA != 0 {

		// When USE_GOMA=true is set and the rule is supported by goma, allow jobs to run outside the local pool.

	} else if ctx.Config().UseRBE() && r.remoteable&SUPPORTS_RBE != 0 {

		// When USE_GOMA=true is set and the rule is supported by RBE, allow jobs to run outside the local pool.

	} else if r.highmem {

		pool = highmemPool

	} else if ctx.Config().UseRemoteBuild() {

		pool = localPool

	}



	ctx.Build(pctx, BuildParams{

		Rule: ctx.Rule(pctx, name, blueprint.RuleParams{

			Command:        commandString,
@@ -408,6 +434,7 @@ func (r *RuleBuilder) Build(pctx PackageContext, ctx BuilderContext, name string 
			Restat:         r.restat,

			Rspfile:        rspFile,

			RspfileContent: rspFileContent,

			Pool:           pool,

		}),

		Inputs:          rspFileInputs,

		Implicits:       r.Inputs(),