Merge changes I07b0ca98,I1c2901e1 (4ddcd815) · Commits · e / os / android_build_soong

cmd/multiproduct_kati/Android.bp

+1 −1

Original line number	Diff line number	Diff line
		@@ -19,8 +19,8 @@ package {
		blueprint_go_binary {
		name: "multiproduct_kati",
		deps: [
		"soong-ui-build",
		"soong-ui-logger",
		"soong-ui-signal",
		"soong-ui-terminal",
		"soong-ui-tracer",
		"soong-zip",

cmd/multiproduct_kati/main.go

+88 −111

Original line number	Diff line number	Diff line
		@@ -20,20 +20,19 @@ import (
		"flag"
		"fmt"
		"io"
		"io/ioutil"
		"log"
		"os"
		"os/exec"
		"path/filepath"
		"regexp"
		"runtime"
		"strings"
		"sync"
		"syscall"
		"time"

		"android/soong/finder"
		"android/soong/ui/build"
		"android/soong/ui/logger"
		"android/soong/ui/signal"
		"android/soong/ui/status"
		"android/soong/ui/terminal"
		"android/soong/ui/tracer"
		@@ -77,36 +76,6 @@ func (m *multipleStringArg) Set(s string) error {
		return nil
		}

		const errorLeadingLines = 20
		const errorTrailingLines = 20

		func errMsgFromLog(filename string) string {
		if filename == "" {
		return ""
		}

		data, err := ioutil.ReadFile(filename)
		if err != nil {
		return ""
		}

		lines := strings.Split(strings.TrimSpace(string(data)), "\n")
		if len(lines) > errorLeadingLines+errorTrailingLines+1 {
		lines[errorLeadingLines] = fmt.Sprintf("... skipping %d lines ...",
		len(lines)-errorLeadingLines-errorTrailingLines)

		lines = append(lines[:errorLeadingLines+1],
		lines[len(lines)-errorTrailingLines:]...)
		}
		var buf strings.Builder
		for _, line := range lines {
		buf.WriteString("> ")
		buf.WriteString(line)
		buf.WriteString("\n")
		}
		return buf.String()
		}

		// TODO(b/70370883): This tool uses a lot of open files -- over the default
		// soft limit of 1024 on some systems. So bump up to the hard limit until I fix
		// the algorithm.
		@@ -158,28 +127,59 @@ func copyFile(from, to string) error {
		}

		type mpContext struct {
		Context context.Context
		Logger logger.Logger
		Status status.ToolStatus
		Tracer tracer.Tracer
		Finder *finder.Finder
		Config build.Config

		LogsDir string
		SoongUi string
		MainOutDir string
		MainLogsDir string
		}

		func detectTotalRAM() uint64 {
		var info syscall.Sysinfo_t
		err := syscall.Sysinfo(&info)
		if err != nil {
		panic(err)
		}
		return info.Totalram * uint64(info.Unit)
		}

		func findNamedProducts(soongUi string, log logger.Logger) []string {
		cmd := exec.Command(soongUi, "--dumpvars-mode", "--vars=all_named_products")
		output, err := cmd.Output()
		if err != nil {
		log.Fatalf("Cannot determine named products: %v", err)
		}

		rx := regexp.MustCompile(`^all_named_products='(.*)'$`)
		match := rx.FindStringSubmatch(strings.TrimSpace(string(output)))
		return strings.Fields(match[1])
		}

		// ensureEmptyFileExists ensures that the containing directory exists, and the
		// specified file exists. If it doesn't exist, it will write an empty file.
		func ensureEmptyFileExists(file string, log logger.Logger) {
		if _, err := os.Stat(file); os.IsNotExist(err) {
		f, err := os.Create(file)
		if err != nil {
		log.Fatalf("Error creating %s: %q\n", file, err)
		}
		f.Close()
		} else if err != nil {
		log.Fatalf("Error checking %s: %q\n", file, err)
		}
		}

		func main() {
		stdio := terminal.StdioImpl{}

		output := terminal.NewStatusOutput(stdio.Stdout(), "", false,
		build.OsEnvironment().IsEnvTrue("ANDROID_QUIET_BUILD"))

		output := terminal.NewStatusOutput(stdio.Stdout(), "", false, false)
		log := logger.New(output)
		defer log.Cleanup()

		flag.Parse()

		ctx, cancel := context.WithCancel(context.Background())
		_, cancel := context.WithCancel(context.Background())
		defer cancel()

		trace := tracer.New(log)
		@@ -192,69 +192,59 @@ func main() {
		var failures failureCount
		stat.AddOutput(&failures)

		build.SetupSignals(log, cancel, func() {
		signal.SetupSignals(log, cancel, func() {
		trace.Close()
		log.Cleanup()
		stat.Finish()
		})

		buildCtx := build.Context{ContextImpl: &build.ContextImpl{
		Context: ctx,
		Logger: log,
		Tracer: trace,
		Writer: output,
		Status: stat,
		}}

		args := ""
		if *alternateResultDir {
		args = "dist"
		}

		originalOutDir := os.Getenv("OUT_DIR")
		if originalOutDir == "" {
		originalOutDir = "out"
		}

		soongUi := "build/soong/soong_ui.bash"

		config := build.NewConfig(buildCtx, args)
		if *outDir == "" {
		var outputDir string
		if *outDir != "" {
		outputDir = *outDir
		} else {
		name := "multiproduct"
		if !*incremental {
		name += "-" + time.Now().Format("20060102150405")
		}

		*outDir = filepath.Join(originalOutDir, name)

		// Ensure the empty files exist in the output directory
		// containing our output directory too. This is mostly for
		// safety, but also triggers the ninja_build file so that our
		// build servers know that they can parse the output as if it
		// was ninja output.
		build.SetupOutDir(buildCtx, config)
		outDirBase := os.Getenv("OUT_DIR")
		if outDirBase == "" {
		outDirBase = "out"
		}

		if err := os.MkdirAll(*outDir, 0777); err != nil {
		log.Fatalf("Failed to create tempdir: %v", err)
		outputDir = filepath.Join(outDirBase, name)
		}

		log.Println("Output directory:", outputDir)

		// The ninja_build file is used by our buildbots to understand that the output
		// can be parsed as ninja output.
		if err := os.MkdirAll(outputDir, 0777); err != nil {
		log.Fatalf("Failed to create output directory: %v", err)
		}
		config.Environment().Set("OUT_DIR", *outDir)
		log.Println("Output directory:", *outDir)
		ensureEmptyFileExists(filepath.Join(outputDir, "ninja_build"), log)

		logsDir := filepath.Join(config.OutDir(), "logs")
		logsDir := filepath.Join(outputDir, "logs")
		os.MkdirAll(logsDir, 0777)

		build.SetupOutDir(buildCtx, config)
		var configLogsDir string
		if *alternateResultDir {
		configLogsDir = filepath.Join(outputDir, "dist/logs")
		} else {
		configLogsDir = outputDir
		}

		os.MkdirAll(config.LogsDir(), 0777)
		log.SetOutput(filepath.Join(config.LogsDir(), "soong.log"))
		trace.SetOutput(filepath.Join(config.LogsDir(), "build.trace"))
		os.MkdirAll(configLogsDir, 0777)
		log.SetOutput(filepath.Join(configLogsDir, "soong.log"))
		trace.SetOutput(filepath.Join(configLogsDir, "build.trace"))

		var jobs = *numJobs
		if jobs < 1 {
		jobs = runtime.NumCPU() / 4

		ramGb := int(config.TotalRAM() / 1024 / 1024 / 1024)
		ramGb := int(detectTotalRAM() / (1024 * 1024 * 1024))
		if ramJobs := ramGb / 25; ramGb > 0 && jobs > ramJobs {
		jobs = ramJobs
		}
		@@ -267,17 +257,8 @@ func main() {

		setMaxFiles(log)

		finder := build.NewSourceFinder(buildCtx, config)
		defer finder.Shutdown()

		build.FindSources(buildCtx, config, finder)

		vars, err := build.DumpMakeVars(buildCtx, config, nil, []string{"all_named_products"})
		if err != nil {
		log.Fatal(err)
		}
		allProducts := findNamedProducts(soongUi, log)
		var productsList []string
		allProducts := strings.Fields(vars["all_named_products"])

		if len(includeProducts) > 0 {
		var missingProducts []string
		@@ -325,19 +306,15 @@ func main() {

		log.Verbose("Got product list: ", finalProductsList)

		s := buildCtx.Status.StartTool()
		s := stat.StartTool()
		s.SetTotalActions(len(finalProductsList))

		mpCtx := &mpContext{
		Context: ctx,
		Logger: log,
		Status: s,
		Tracer: trace,

		Finder: finder,
		Config: config,

		LogsDir: logsDir,
		SoongUi: soongUi,
		MainOutDir: outputDir,
		MainLogsDir: logsDir,
		}

		products := make(chan string, len(productsList))
		@@ -359,7 +336,7 @@ func main() {
		if product == "" {
		return
		}
		runSoongUiForProduct(mpCtx, product, soongUi)
		runSoongUiForProduct(mpCtx, product)
		}
		}
		}()
		@@ -371,7 +348,7 @@ func main() {
		FileArgs: []zip.FileArg{
		{GlobDir: logsDir, SourcePrefixToStrip: logsDir},
		},
		OutputFilePath: filepath.Join(config.RealDistDir(), "logs.zip"),
		OutputFilePath: filepath.Join(outputDir, "dist/logs.zip"),
		NumParallelJobs: runtime.NumCPU(),
		CompressionLevel: 5,
		}
		@@ -413,10 +390,10 @@ func cleanupAfterProduct(outDir, productZip string) {
		}
		}

		func runSoongUiForProduct(mpctx *mpContext, product, soongUi string) {
		outDir := filepath.Join(mpctx.Config.OutDir(), product)
		logsDir := filepath.Join(mpctx.LogsDir, product)
		productZip := filepath.Join(mpctx.Config.OutDir(), product+".zip")
		func runSoongUiForProduct(mpctx *mpContext, product string) {
		outDir := filepath.Join(mpctx.MainOutDir, product)
		logsDir := filepath.Join(mpctx.MainLogsDir, product)
		productZip := filepath.Join(mpctx.MainOutDir, product+".zip")
		consoleLogPath := filepath.Join(logsDir, "std.log")

		if err := os.MkdirAll(outDir, 0777); err != nil {
		@@ -451,7 +428,7 @@ func runSoongUiForProduct(mpctx *mpContext, product, soongUi string) {
		args = append(args, "dist")
		}

		cmd := exec.Command(soongUi, args...)
		cmd := exec.Command(mpctx.SoongUi, args...)
		cmd.Stdout = consoleLogWriter
		cmd.Stderr = consoleLogWriter
		cmd.Env = append(os.Environ(),

cmd/soong_ui/Android.bp

+1 −0

Original line number	Diff line number	Diff line
		@@ -20,6 +20,7 @@ blueprint_go_binary {
		name: "soong_ui",
		deps: [
		"soong-ui-build",
		"soong-ui-signal",
		"soong-ui-logger",
		"soong-ui-terminal",
		"soong-ui-tracer",

cmd/soong_ui/main.go

+2 −1

Original line number	Diff line number	Diff line
		@@ -30,6 +30,7 @@ import (
		"android/soong/ui/build"
		"android/soong/ui/logger"
		"android/soong/ui/metrics"
		"android/soong/ui/signal"
		"android/soong/ui/status"
		"android/soong/ui/terminal"
		"android/soong/ui/tracer"
		@@ -190,7 +191,7 @@ func main() {
		stat.AddOutput(trace.StatusTracer())

		// Set up a cleanup procedure in case the normal termination process doesn't work.
		build.SetupSignals(log, cancel, func() {
		signal.SetupSignals(log, cancel, func() {
		trace.Close()
		log.Cleanup()
		stat.Finish()

cuj/Android.bp

+1 −0

Original line number	Diff line number	Diff line
		@@ -7,6 +7,7 @@ blueprint_go_binary {
		deps: [
		"soong-ui-build",
		"soong-ui-logger",
		"soong-ui-signal",
		"soong-ui-terminal",
		"soong-ui-tracer",
		],