[DO NOT MERGE] Add RBE metrics dump in Soong UI. am: fbc4e3f9b6 (13ea2a5d) · Commits · e / os / android_build_soong

cmd/soong_ui/main.go

+1 −0

Original line number	Diff line number	Diff line
		@@ -131,6 +131,7 @@ func main() {
		stat.AddOutput(status.NewErrorLog(log, filepath.Join(logsDir, "error.log")))

		defer met.Dump(soongMetricsFile)
		defer build.DumpRBEMetrics(buildCtx, config, rbeMetricsFile)

		if start, ok := os.LookupEnv("TRACE_BEGIN_SOONG"); ok {
		if !strings.HasSuffix(start, "N") {

ui/build/Android.bp

+1 −0

Original line number	Diff line number	Diff line
		@@ -62,6 +62,7 @@ bootstrap_go_package {
		testSrcs: [
		"config_test.go",
		"environment_test.go",
		"rbe_test.go",
		"upload_test.go",
		"util_test.go",
		"proc_sync_test.go",

ui/build/config.go

+13 −0

Original line number	Diff line number	Diff line
		@@ -522,6 +522,19 @@ func (c *configImpl) StartRBE() bool {
		return true
		}

		func (c *configImpl) RBEStatsOutputDir() string {
		for _, f := range []string{"RBE_output_dir", "FLAG_output_dir"} {
		if v, ok := c.environ.Get(f); ok {
		return v
		}
		}
		return ""
		}

		func (c *configImpl) UseRemoteBuild() bool {
		return c.UseGoma() \|\| c.UseRBE()
		}

		// RemoteParallel controls how many remote jobs (i.e., commands which contain
		// gomacc) are run in parallel. Note the parallelism of all other jobs is
		// still limited by Parallel()

ui/build/rbe.go

+70 −13

Original line number	Diff line number	Diff line
		@@ -17,15 +17,40 @@ package build
		import (
		"fmt"
		"math/rand"
		"os"
		"path/filepath"
		"time"

		"android/soong/ui/metrics"
		)

		const bootstrapCmd = "bootstrap"
		const rbeLeastNProcs = 2500
		const rbeLeastNFiles = 16000
		const (
		rbeLeastNProcs = 2500
		rbeLeastNFiles = 16000

		// prebuilt RBE binaries
		bootstrapCmd = "bootstrap"

		// RBE metrics proto buffer file
		rbeMetricsPBFilename = "rbe_metrics.pb"
		)

		func rbeCommand(ctx Context, config Config, rbeCmd string) string {
		var cmdPath string
		if rbeDir, ok := config.Environment().Get("RBE_DIR"); ok {
		cmdPath = filepath.Join(rbeDir, rbeCmd)
		} else if home, ok := config.Environment().Get("HOME"); ok {
		cmdPath = filepath.Join(home, "rbe", rbeCmd)
		} else {
		ctx.Fatalf("rbe command path not found")
		}

		if _, err := os.Stat(cmdPath); err != nil && os.IsNotExist(err) {
		ctx.Fatalf("rbe command %q not found", rbeCmd)
		}

		return cmdPath
		}

		func getRBEVars(ctx Context, tmpDir string) map[string]string {
		rand.Seed(time.Now().UnixNano())
		@@ -43,18 +68,50 @@ func startRBE(ctx Context, config Config) {
		ctx.Fatalf("max open files is insufficient: %d; want >= %d.\n", n, rbeLeastNFiles)
		}

		var rbeBootstrap string
		if rbeDir, ok := config.Environment().Get("RBE_DIR"); ok {
		rbeBootstrap = filepath.Join(rbeDir, bootstrapCmd)
		} else if home, ok := config.Environment().Get("HOME"); ok {
		rbeBootstrap = filepath.Join(home, "rbe", bootstrapCmd)
		} else {
		ctx.Fatalln("rbe bootstrap not found")
		}

		cmd := Command(ctx, config, "bootstrap", rbeBootstrap)
		cmd := Command(ctx, config, "startRBE bootstrap", rbeCommand(ctx, config, bootstrapCmd))

		if output, err := cmd.CombinedOutput(); err != nil {
		ctx.Fatalf("rbe bootstrap failed with: %v\n%s\n", err, output)
		}
		}

		func stopRBE(ctx Context, config Config) {
		cmd := Command(ctx, config, "stopRBE bootstrap", rbeCommand(ctx, config, bootstrapCmd), "-shutdown")
		if output, err := cmd.CombinedOutput(); err != nil {
		ctx.Fatalf("rbe bootstrap with shutdown failed with: %v\n%s\n", err, output)
		}
		}

		// DumpRBEMetrics creates a metrics protobuf file containing RBE related metrics.
		// The protobuf file is created if RBE is enabled and the proxy service has
		// started. The proxy service is shutdown in order to dump the RBE metrics to the
		// protobuf file.
		func DumpRBEMetrics(ctx Context, config Config, filename string) {
		ctx.BeginTrace(metrics.RunShutdownTool, "dump_rbe_metrics")
		defer ctx.EndTrace()

		// Remove the previous metrics file in case there is a failure or RBE has been
		// disable for this run.
		os.Remove(filename)

		// If RBE is not enabled then there are no metrics to generate.
		// If RBE does not require to start, the RBE proxy maybe started
		// manually for debugging purpose and can generate the metrics
		// afterwards.
		if !config.StartRBE() {
		return
		}

		outputDir := config.RBEStatsOutputDir()
		if outputDir == "" {
		ctx.Fatal("RBE output dir variable not defined. Aborting metrics dumping.")
		}
		metricsFile := filepath.Join(outputDir, rbeMetricsPBFilename)

		// Stop the proxy first in order to generate the RBE metrics protobuf file.
		stopRBE(ctx, config)

		if _, err := copyFile(metricsFile, filename); err != nil {
		ctx.Fatalf("failed to copy %q to %q: %v\n", metricsFile, filename, err)
		}
		}

ui/build/rbe_test.go

0 → 100644

+150 −0

Original line number	Diff line number	Diff line
		// Copyright 2020 Google Inc. All rights reserved.
		//
		// Licensed under the Apache License, Version 2.0 (the "License");
		// you may not use this file except in compliance with the License.
		// You may obtain a copy of the License at
		//
		// http://www.apache.org/licenses/LICENSE-2.0
		//
		// Unless required by applicable law or agreed to in writing, software
		// distributed under the License is distributed on an "AS IS" BASIS,
		// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
		// See the License for the specific language governing permissions and
		// limitations under the License.

		package build

		import (
		"fmt"
		"io/ioutil"
		"os"
		"path/filepath"
		"strings"
		"testing"

		"android/soong/ui/logger"
		)

		func TestDumpRBEMetrics(t *testing.T) {
		ctx := testContext()
		tests := []struct {
		description string
		env []string
		generated bool
		}{{
		description: "RBE disabled",
		env: []string{
		"NOSTART_RBE=true",
		},
		}, {
		description: "rbe metrics generated",
		env: []string{
		"USE_RBE=true",
		},
		generated: true,
		}}

		for _, tt := range tests {
		t.Run(tt.description, func(t *testing.T) {
		tmpDir, err := ioutil.TempDir("", "tmpdir")
		if err != nil {
		t.Fatalf("failed to make temp dir: %v", err)
		}
		defer os.RemoveAll(tmpDir)

		rbeBootstrapCmd := filepath.Join(tmpDir, bootstrapCmd)
		if err := ioutil.WriteFile(rbeBootstrapCmd, []byte(rbeBootstrapProgram), 0755); err != nil {
		t.Fatalf("failed to create a fake bootstrap command file %s: %v", rbeBootstrapCmd, err)
		}

		env := Environment(tt.env)
		env.Set("OUT_DIR", tmpDir)
		env.Set("RBE_DIR", tmpDir)
		env.Set("RBE_output_dir", tmpDir)
		config := Config{&configImpl{
		environ: &env,
		}}

		rbeMetricsFilename := filepath.Join(tmpDir, rbeMetricsPBFilename)
		DumpRBEMetrics(ctx, config, rbeMetricsFilename)

		// Validate that the rbe metrics file exists if RBE is enabled.
		if _, err := os.Stat(rbeMetricsFilename); err == nil {
		if !tt.generated {
		t.Errorf("got true, want false for rbe metrics file %s to exist.", rbeMetricsFilename)
		}
		} else if os.IsNotExist(err) {
		if tt.generated {
		t.Errorf("got false, want true for rbe metrics file %s to exist.", rbeMetricsFilename)
		}
		} else {
		t.Errorf("unknown error found on checking %s exists: %v", rbeMetricsFilename, err)
		}
		})
		}
		}

		func TestDumpRBEMetricsErrors(t *testing.T) {
		ctx := testContext()
		tests := []struct {
		description string
		rbeOutputDirDefined bool
		bootstrapProgram string
		expectedErr string
		}{{
		description: "output_dir not defined",
		bootstrapProgram: rbeBootstrapProgram,
		expectedErr: "RBE output dir variable not defined",
		}, {
		description: "stopRBE failed",
		rbeOutputDirDefined: true,
		bootstrapProgram: "#!/bin/bash\nexit 1",
		expectedErr: "shutdown failed",
		}, {
		description: "failed to copy metrics file",
		rbeOutputDirDefined: true,
		bootstrapProgram: "#!/bin/bash",
		expectedErr: "failed to copy",
		}}

		for _, tt := range tests {
		t.Run(tt.description, func(t *testing.T) {
		defer logger.Recover(func(err error) {
		got := err.Error()
		if !strings.Contains(got, tt.expectedErr) {
		t.Errorf("got %q, want %q to be contained in error", got, tt.expectedErr)
		}
		})

		tmpDir, err := ioutil.TempDir("", "tmpdir")
		if err != nil {
		t.Fatalf("failed to make temp dir: %v", err)
		}
		defer os.RemoveAll(tmpDir)

		rbeBootstrapCmd := filepath.Join(tmpDir, bootstrapCmd)
		if err := ioutil.WriteFile(rbeBootstrapCmd, []byte(tt.bootstrapProgram), 0755); err != nil {
		t.Fatalf("failed to create a fake bootstrap command file %s: %v", rbeBootstrapCmd, err)
		}

		env := &Environment{}
		env.Set("USE_RBE", "true")
		env.Set("OUT_DIR", tmpDir)
		env.Set("RBE_DIR", tmpDir)

		if tt.rbeOutputDirDefined {
		env.Set("RBE_output_dir", tmpDir)
		}

		config := Config{&configImpl{
		environ: env,
		}}

		rbeMetricsFilename := filepath.Join(tmpDir, rbeMetricsPBFilename)
		DumpRBEMetrics(ctx, config, rbeMetricsFilename)
		t.Errorf("got nil, expecting %q as a failure", tt.expectedErr)
		})
		}
		}

		var rbeBootstrapProgram = fmt.Sprintf("#!/bin/bash\necho 1 > $RBE_output_dir/%s", rbeMetricsPBFilename)