Merge "Experimental code to support build action caching." into main (88f4e331) · Commits · e / os / android_build_soong

aconfig/aconfig_declarations.go

+15 −0

Original line number	Diff line number	Diff line
		@@ -32,6 +32,7 @@ type AconfigReleaseConfigValue struct {
		type DeclarationsModule struct {
		android.ModuleBase
		android.DefaultableModuleBase
		blueprint.IncrementalModule

		// Properties for "aconfig_declarations"
		properties struct {
		@@ -217,3 +218,17 @@ func (module *DeclarationsModule) GenerateAndroidBuildActions(ctx android.Module
		android.SetProvider(ctx, android.AconfigDeclarationsProviderKey, providerData[""])
		android.SetProvider(ctx, android.AconfigReleaseDeclarationsProviderKey, providerData)
		}

		func (module *DeclarationsModule) BuildActionProviderKeys() []blueprint.AnyProviderKey {
		return []blueprint.AnyProviderKey{android.AconfigDeclarationsProviderKey}
		}

		func (module *DeclarationsModule) PackageContextPath() string {
		return pkgPath
		}

		func (module *DeclarationsModule) CachedRules() []blueprint.Rule {
		return []blueprint.Rule{aconfigRule, aconfigTextRule}
		}

		var _ blueprint.Incremental = &DeclarationsModule{}

aconfig/init.go

+7 −1

Original line number	Diff line number	Diff line
		@@ -15,13 +15,16 @@
		package aconfig

		import (
		"encoding/gob"

		"android/soong/android"

		"github.com/google/blueprint"
		)

		var (
		pctx = android.NewPackageContext("android/soong/aconfig")
		pkgPath = "android/soong/aconfig"
		pctx = android.NewPackageContext(pkgPath)

		// For aconfig_declarations: Generate cache file
		aconfigRule = pctx.AndroidStaticRule("aconfig",
		@@ -106,6 +109,9 @@ func init() {
		RegisterBuildComponents(android.InitRegistrationContext)
		pctx.HostBinToolVariable("aconfig", "aconfig")
		pctx.HostBinToolVariable("soong_zip", "soong_zip")

		gob.Register(android.AconfigDeclarationsProviderData{})
		gob.Register(android.ModuleOutPath{})
		}

		func RegisterBuildComponents(ctx android.RegistrationContext) {

android/module.go

+48 −3

Original line number	Diff line number	Diff line
		@@ -1913,10 +1913,55 @@ func (m *ModuleBase) GenerateBuildActions(blueprintCtx blueprint.ModuleContext)
		return
		}

		incrementalAnalysis := false
		incrementalEnabled := false
		var cacheKey *blueprint.BuildActionCacheKey = nil
		var incrementalModule *blueprint.Incremental = nil
		if ctx.bp.GetIncrementalEnabled() {
		if im, ok := m.module.(blueprint.Incremental); ok {
		incrementalModule = &im
		incrementalEnabled = im.IncrementalSupported()
		incrementalAnalysis = ctx.bp.GetIncrementalAnalysis() && incrementalEnabled
		}
		}
		if incrementalEnabled {
		hash, err := proptools.CalculateHash(m.GetProperties())
		if err != nil {
		ctx.ModuleErrorf("failed to calculate properties hash: %s", err)
		return
		}
		cacheInput := new(blueprint.BuildActionCacheInput)
		cacheInput.PropertiesHash = hash
		ctx.VisitDirectDeps(func(module Module) {
		cacheInput.ProvidersHash =
		append(cacheInput.ProvidersHash, ctx.bp.OtherModuleProviderInitialValueHashes(module))
		})
		hash, err = proptools.CalculateHash(&cacheInput)
		if err != nil {
		ctx.ModuleErrorf("failed to calculate cache input hash: %s", err)
		return
		}
		cacheKey = &blueprint.BuildActionCacheKey{
		Id: ctx.bp.ModuleId(),
		InputHash: hash,
		}
		}

		restored := false
		if incrementalAnalysis && cacheKey != nil {
		restored = ctx.bp.RestoreBuildActions(cacheKey, incrementalModule)
		}

		if !restored {
		m.module.GenerateAndroidBuildActions(ctx)
		if ctx.Failed() {
		return
		}
		}

		if incrementalEnabled && cacheKey != nil {
		ctx.bp.CacheBuildActions(cacheKey, incrementalModule)
		}

		// Create the set of tagged dist files after calling GenerateAndroidBuildActions
		// as GenerateTaggedDistFiles() calls OutputFiles(tag) and so relies on the

android/paths.go

+47 −0

Original line number	Diff line number	Diff line
		@@ -15,6 +15,9 @@
		package android

		import (
		"bytes"
		"encoding/gob"
		"errors"
		"fmt"
		"os"
		"path/filepath"
		@@ -1068,6 +1071,28 @@ type basePath struct {
		rel string
		}

		func (p basePath) GobEncode() ([]byte, error) {
		w := new(bytes.Buffer)
		encoder := gob.NewEncoder(w)
		err := errors.Join(encoder.Encode(p.path), encoder.Encode(p.rel))
		if err != nil {
		return nil, err
		}

		return w.Bytes(), nil
		}

		func (p *basePath) GobDecode(data []byte) error {
		r := bytes.NewBuffer(data)
		decoder := gob.NewDecoder(r)
		err := errors.Join(decoder.Decode(&p.path), decoder.Decode(&p.rel))
		if err != nil {
		return err
		}

		return nil
		}

		func (p basePath) Ext() string {
		return filepath.Ext(p.path)
		}
		@@ -1306,6 +1331,28 @@ type OutputPath struct {
		fullPath string
		}

		func (p OutputPath) GobEncode() ([]byte, error) {
		w := new(bytes.Buffer)
		encoder := gob.NewEncoder(w)
		err := errors.Join(encoder.Encode(p.basePath), encoder.Encode(p.soongOutDir), encoder.Encode(p.fullPath))
		if err != nil {
		return nil, err
		}

		return w.Bytes(), nil
		}

		func (p *OutputPath) GobDecode(data []byte) error {
		r := bytes.NewBuffer(data)
		decoder := gob.NewDecoder(r)
		err := errors.Join(decoder.Decode(&p.basePath), decoder.Decode(&p.soongOutDir), decoder.Decode(&p.fullPath))
		if err != nil {
		return err
		}

		return nil
		}

		func (p OutputPath) withRel(rel string) OutputPath {
		p.basePath = p.basePath.withRel(rel)
		p.fullPath = filepath.Join(p.fullPath, rel)

cmd/soong_build/main.go

+83 −1

Original line number	Diff line number	Diff line
		@@ -16,6 +16,7 @@ package main

		import (
		"bytes"
		"encoding/json"
		"errors"
		"flag"
		"fmt"
		@@ -28,11 +29,11 @@ import (
		"android/soong/android/allowlists"
		"android/soong/bp2build"
		"android/soong/shared"

		"github.com/google/blueprint"
		"github.com/google/blueprint/bootstrap"
		"github.com/google/blueprint/deptools"
		"github.com/google/blueprint/metrics"
		"github.com/google/blueprint/proptools"
		androidProtobuf "google.golang.org/protobuf/android"
		)

		@@ -49,6 +50,14 @@ var (
		cmdlineArgs android.CmdArgs
		)

		const configCacheFile = "config.cache"

		type ConfigCache struct {
		EnvDepsHash uint64
		ProductVariableFileTimestamp int64
		SoongBuildFileTimestamp int64
		}

		func init() {
		// Flags that make sense in every mode
		flag.StringVar(&topDir, "top", "", "Top directory of the Android source tree")
		@@ -82,6 +91,7 @@ func init() {
		// Flags that probably shouldn't be flags of soong_build, but we haven't found
		// the time to remove them yet
		flag.BoolVar(&cmdlineArgs.RunGoTests, "t", false, "build and run go tests during bootstrap")
		flag.BoolVar(&cmdlineArgs.IncrementalBuildActions, "incremental-build-actions", false, "generate build actions incrementally")

		// Disable deterministic randomization in the protobuf package, so incremental
		// builds with unrelated Soong changes don't trigger large rebuilds (since we
		@@ -218,6 +228,60 @@ func writeDepFile(outputFile string, eventHandler *metrics.EventHandler, ninjaDe
		maybeQuit(err, "error writing depfile '%s'", depFile)
		}

		// Check if there are changes to the environment file, product variable file and
		// soong_build binary, in which case no incremental will be performed.
		func incrementalValid(config android.Config, configCacheFile string) (*ConfigCache, bool) {
		var newConfigCache ConfigCache
		data, err := os.ReadFile(shared.JoinPath(topDir, usedEnvFile))
		if err != nil {
		// Clean build
		if os.IsNotExist(err) {
		data = []byte{}
		} else {
		maybeQuit(err, "")
		}
		}

		newConfigCache.EnvDepsHash, err = proptools.CalculateHash(data)
		newConfigCache.ProductVariableFileTimestamp = getFileTimestamp(filepath.Join(topDir, cmdlineArgs.SoongVariables))
		newConfigCache.SoongBuildFileTimestamp = getFileTimestamp(filepath.Join(topDir, config.HostToolDir(), "soong_build"))
		//TODO(b/344917959): out/soong/dexpreopt.config might need to be checked as well.

		file, err := os.Open(configCacheFile)
		if err != nil && os.IsNotExist(err) {
		return &newConfigCache, false
		}
		maybeQuit(err, "")
		defer file.Close()

		var configCache ConfigCache
		decoder := json.NewDecoder(file)
		err = decoder.Decode(&configCache)
		maybeQuit(err, "")

		return &newConfigCache, newConfigCache == configCache
		}

		func getFileTimestamp(file string) int64 {
		stat, err := os.Stat(file)
		if err == nil {
		return stat.ModTime().UnixMilli()
		} else if !os.IsNotExist(err) {
		maybeQuit(err, "")
		}
		return 0
		}

		func writeConfigCache(configCache *ConfigCache, configCacheFile string) {
		file, err := os.Create(configCacheFile)
		maybeQuit(err, "")
		defer file.Close()

		encoder := json.NewEncoder(file)
		err = encoder.Encode(*configCache)
		maybeQuit(err, "")
		}

		// runSoongOnlyBuild runs the standard Soong build in a number of different modes.
		func runSoongOnlyBuild(ctx *android.Context, extraNinjaDeps []string) string {
		ctx.EventHandler.Begin("soong_build")
		@@ -319,8 +383,26 @@ func main() {
		ctx := newContext(configuration)
		android.StartBackgroundMetrics(configuration)

		var configCache *ConfigCache
		configFile := filepath.Join(topDir, ctx.Config().OutDir(), configCacheFile)
		incremental := false
		ctx.SetIncrementalEnabled(cmdlineArgs.IncrementalBuildActions)
		if cmdlineArgs.IncrementalBuildActions {
		configCache, incremental = incrementalValid(ctx.Config(), configFile)
		}
		ctx.SetIncrementalAnalysis(incremental)

		ctx.Register()
		finalOutputFile := runSoongOnlyBuild(ctx, extraNinjaDeps)

		if ctx.GetIncrementalEnabled() {
		data, err := shared.EnvFileContents(configuration.EnvDeps())
		maybeQuit(err, "")
		configCache.EnvDepsHash, err = proptools.CalculateHash(data)
		maybeQuit(err, "")
		writeConfigCache(configCache, configFile)
		}

		writeMetrics(configuration, ctx.EventHandler, metricsDir)

		writeUsedEnvironmentFile(configuration)