Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit cef87b62 authored by Lukacs T. Berki's avatar Lukacs T. Berki
Browse files

Make multiproduct_kati call soong_ui.bash . 

This serves to not link parts of soong_ui (and eventually soong_build)
into a separate, weird binary. This is in turn good because they contain
any number of global variables and no one really thought about what
happens when two instances are executing at the same time in the same
address space.

This comes with a slight performance hit: 5 aosp_* projects build 152
seconds instead of 146. I suppose this is a price worth paying for a
clean design?

Test: presubmits.
Change-Id: I5623dcab2290f0fc392dd2ede597b9794a3d2a4e
parent 01cad0cd

cmd/multiproduct_kati/main.go

+76 −76
Original line number Diff line number Diff line
@@ -15,12 +15,15 @@ 
package main



import (

	"bufio"

	"context"

	"flag"

	"fmt"

	"io"

	"io/ioutil"

	"log"

	"os"

	"os/exec"

	"path/filepath"

	"runtime"

	"strings"
@@ -207,6 +210,14 @@ func main() { 
	if *alternateResultDir {

		args = "dist"

	}



	originalOutDir := os.Getenv("OUT_DIR")

	if originalOutDir == "" {

		originalOutDir = "out"

	}



	soongUi := "build/soong/soong_ui.bash"



	config := build.NewConfig(buildCtx, args)

	if *outDir == "" {

		name := "multiproduct"
@@ -214,7 +225,7 @@ func main() { 
			name += "-" + time.Now().Format("20060102150405")

		}



		*outDir = filepath.Join(config.OutDir(), name)

		*outDir = filepath.Join(originalOutDir, name)



		// Ensure the empty files exist in the output directory

		// containing our output directory too. This is mostly for
@@ -348,7 +359,7 @@ func main() { 
					if product == "" {

						return

					}

					buildProduct(mpCtx, product)

					runSoongUiForProduct(mpCtx, product, soongUi)

				}

			}

		}()
@@ -380,11 +391,33 @@ func main() { 
	}

}



func buildProduct(mpctx *mpContext, product string) {

	var stdLog string

func cleanupAfterProduct(outDir, productZip string) {

	if *keepArtifacts {

		args := zip.ZipArgs{

			FileArgs: []zip.FileArg{

				{

					GlobDir:             outDir,

					SourcePrefixToStrip: outDir,

				},

			},

			OutputFilePath:   productZip,

			NumParallelJobs:  runtime.NumCPU(),

			CompressionLevel: 5,

		}

		if err := zip.Zip(args); err != nil {

			log.Fatalf("Error zipping artifacts: %v", err)

		}

	}

	if !*incremental {

		os.RemoveAll(outDir)

	}

}



func runSoongUiForProduct(mpctx *mpContext, product, soongUi string) {

	outDir := filepath.Join(mpctx.Config.OutDir(), product)

	logsDir := filepath.Join(mpctx.LogsDir, product)

	productZip := filepath.Join(mpctx.Config.OutDir(), product+".zip")

	consoleLogPath := filepath.Join(logsDir, "std.log")



	if err := os.MkdirAll(outDir, 0777); err != nil {

		mpctx.Logger.Fatalf("Error creating out directory: %v", err)
@@ -393,98 +426,65 @@ func buildProduct(mpctx *mpContext, product string) { 
		mpctx.Logger.Fatalf("Error creating log directory: %v", err)

	}



	stdLog = filepath.Join(logsDir, "std.log")

	f, err := os.Create(stdLog)

	consoleLogFile, err := os.Create(consoleLogPath)

	if err != nil {

		mpctx.Logger.Fatalf("Error creating std.log: %v", err)

		mpctx.Logger.Fatalf("Error creating console log file: %v", err)

	}

	defer f.Close()



	log := logger.New(f)

	defer log.Cleanup()

	log.SetOutput(filepath.Join(logsDir, "soong.log"))

	defer consoleLogFile.Close()



	action := &status.Action{

		Description: product,

		Outputs:     []string{product},

	}

	mpctx.Status.StartAction(action)

	defer logger.Recover(func(err error) {

		mpctx.Status.FinishAction(status.ActionResult{

			Action: action,

			Error:  err,

			Output: errMsgFromLog(stdLog),

		})

	})

	consoleLogWriter := bufio.NewWriter(consoleLogFile)

	defer consoleLogWriter.Flush()



	ctx := build.Context{ContextImpl: &build.ContextImpl{

		Context: mpctx.Context,

		Logger:  log,

		Tracer:  mpctx.Tracer,

		Writer:  f,

		Thread:  mpctx.Tracer.NewThread(product),

		Status:  &status.Status{},

	}}

	ctx.Status.AddOutput(terminal.NewStatusOutput(ctx.Writer, "", false,

		build.OsEnvironment().IsEnvTrue("ANDROID_QUIET_BUILD")))

	args := []string{"--make-mode", "--skip-soong-tests", "--skip-ninja"}



	args := append([]string(nil), flag.Args()...)

	args = append(args, "--skip-soong-tests")

	config := build.NewConfig(ctx, args...)

	config.Environment().Set("OUT_DIR", outDir)

	if !*keepArtifacts {

		config.SetEmptyNinjaFile(true)

		args = append(args, "--empty-ninja-file")

	}

	build.FindSources(ctx, config, mpctx.Finder)

	config.Lunch(ctx, product, *buildVariant)



	defer func() {

		if *keepArtifacts {

			args := zip.ZipArgs{

				FileArgs: []zip.FileArg{

					{

						GlobDir:             outDir,

						SourcePrefixToStrip: outDir,

					},

				},

				OutputFilePath:   filepath.Join(mpctx.Config.OutDir(), product+".zip"),

				NumParallelJobs:  runtime.NumCPU(),

				CompressionLevel: 5,

			}

			if err := zip.Zip(args); err != nil {

				log.Fatalf("Error zipping artifacts: %v", err)

	if *onlyConfig {

		args = append(args, "--config-only")

	} else if *onlySoong {

		args = append(args, "--soong-only")

	}



	if *alternateResultDir {

		args = append(args, "dist")

	}

		if !*incremental {

			os.RemoveAll(outDir)

		}

	}()



	config.SetSkipNinja(true)

	cmd := exec.Command(soongUi, args...)

	cmd.Stdout = consoleLogWriter

	cmd.Stderr = consoleLogWriter

	cmd.Env = append(os.Environ(),

		"OUT_DIR="+outDir,

		"TARGET_PRODUCT="+product,

		"TARGET_BUILD_VARIANT="+*buildVariant,

		"TARGET_BUILD_TYPE=release",

		"TARGET_BUILD_APPS=",

		"TARGET_BUILD_UNBUNDLED=")



	buildWhat := build.RunProductConfig

	if !*onlyConfig {

		buildWhat |= build.RunSoong

		if !*onlySoong {

			buildWhat |= build.RunKati

		}

	action := &status.Action{

		Description: product,

		Outputs:     []string{product},

	}



	mpctx.Status.StartAction(action)

	defer cleanupAfterProduct(outDir, productZip)



	before := time.Now()

	build.Build(ctx, config)

	err = cmd.Run()



	// Save std_full.log if Kati re-read the makefiles

	if buildWhat&build.RunKati != 0 {

		if after, err := os.Stat(config.KatiBuildNinjaFile()); err == nil && after.ModTime().After(before) {

			err := copyFile(stdLog, filepath.Join(filepath.Dir(stdLog), "std_full.log"))

	if !*onlyConfig && !*onlySoong {

		katiBuildNinjaFile := filepath.Join(outDir, "build-"+product+".ninja")

		if after, err := os.Stat(katiBuildNinjaFile); err == nil && after.ModTime().After(before) {

			err := copyFile(consoleLogPath, filepath.Join(filepath.Dir(consoleLogPath), "std_full.log"))

			if err != nil {

				log.Fatalf("Error copying log file: %s", err)

			}

		}

	}



	mpctx.Status.FinishAction(status.ActionResult{

		Action: action,

		Error:  err,

	})

}

ui/build/build.go

+5 −0
Original line number Diff line number Diff line
@@ -238,6 +238,11 @@ func Build(ctx Context, config Config) { 
		ctx.Verboseln("Skipping use of Kati ninja as requested")

		what = what &^ RunKatiNinja

	}

	if config.SkipSoong() {

		ctx.Verboseln("Skipping use of Soong as requested")

		what = what &^ RunSoong

	}



	if config.SkipNinja() {

		ctx.Verboseln("Skipping Ninja as requested")

		what = what &^ RunNinja

ui/build/config.go

+11 −48
Original line number Diff line number Diff line
@@ -49,6 +49,7 @@ type configImpl struct { 
	skipConfig     bool

	skipKati       bool

	skipKatiNinja  bool

	skipSoong      bool

	skipNinja      bool

	skipSoongTests bool
@@ -582,6 +583,8 @@ func (c *configImpl) parseArgs(ctx Context, args []string) { 
		arg := strings.TrimSpace(args[i])

		if arg == "showcommands" {

			c.verbose = true

		} else if arg == "--empty-ninja-file" {

			c.emptyNinjaFile = true

		} else if arg == "--skip-ninja" {

			c.skipNinja = true

		} else if arg == "--skip-make" {
@@ -596,6 +599,10 @@ func (c *configImpl) parseArgs(ctx Context, args []string) { 
		} else if arg == "--soong-only" {

			c.skipKati = true

			c.skipKatiNinja = true

		} else if arg == "--config-only" {

			c.skipKati = true

			c.skipKatiNinja = true

			c.skipSoong = true

		} else if arg == "--skip-config" {

			c.skipConfig = true

		} else if arg == "--skip-soong-tests" {
@@ -690,54 +697,6 @@ func (c *configImpl) configureLocale(ctx Context) { 
	}

}



// Lunch configures the environment for a specific product similarly to the

// `lunch` bash function.

func (c *configImpl) Lunch(ctx Context, product, variant string) {

	if variant != "eng" && variant != "userdebug" && variant != "user" {

		ctx.Fatalf("Invalid variant %q. Must be one of 'user', 'userdebug' or 'eng'", variant)

	}



	c.environ.Set("TARGET_PRODUCT", product)

	c.environ.Set("TARGET_BUILD_VARIANT", variant)

	c.environ.Set("TARGET_BUILD_TYPE", "release")

	c.environ.Unset("TARGET_BUILD_APPS")

	c.environ.Unset("TARGET_BUILD_UNBUNDLED")

}



// Tapas configures the environment to build one or more unbundled apps,

// similarly to the `tapas` bash function.

func (c *configImpl) Tapas(ctx Context, apps []string, arch, variant string) {

	if len(apps) == 0 {

		apps = []string{"all"}

	}

	if variant == "" {

		variant = "eng"

	}



	if variant != "eng" && variant != "userdebug" && variant != "user" {

		ctx.Fatalf("Invalid variant %q. Must be one of 'user', 'userdebug' or 'eng'", variant)

	}



	var product string

	switch arch {

	case "arm", "":

		product = "aosp_arm"

	case "arm64":

		product = "aosm_arm64"

	case "x86":

		product = "aosp_x86"

	case "x86_64":

		product = "aosp_x86_64"

	default:

		ctx.Fatalf("Invalid architecture: %q", arch)

	}



	c.environ.Set("TARGET_PRODUCT", product)

	c.environ.Set("TARGET_BUILD_VARIANT", variant)

	c.environ.Set("TARGET_BUILD_TYPE", "release")

	c.environ.Set("TARGET_BUILD_APPS", strings.Join(apps, " "))

}



func (c *configImpl) Environment() *Environment {

	return c.environ

}
@@ -817,6 +776,10 @@ func (c *configImpl) SkipKatiNinja() bool { 
	return c.skipKatiNinja

}



func (c *configImpl) SkipSoong() bool {

	return c.skipSoong

}



func (c *configImpl) SkipNinja() bool {

	return c.skipNinja

}