Upload build metrics after a build is completed. am: 4f7f60b54b am: e2eadc4457 (d6ea19f3) · Commits · e / os / android_build_soong

cmd/soong_ui/main.go

+9 −2

Original line number	Original line	Diff line number	Diff line
	@@ -117,6 +117,8 @@ func inList(s string, list []string) bool {
	// Command is the type of soong_ui execution. Only one type of		// Command is the type of soong_ui execution. Only one type of
	// execution is specified. The args are specific to the command.		// execution is specified. The args are specific to the command.
	func main() {		func main() {
			buildStartedMilli := time.Now().UnixNano() / int64(time.Millisecond)

	c, args := getCommand(os.Args)		c, args := getCommand(os.Args)
	if c == nil {		if c == nil {
	fmt.Fprintf(os.Stderr, "The `soong` native UI is not yet available.\n")		fmt.Fprintf(os.Stderr, "The `soong` native UI is not yet available.\n")
	@@ -166,19 +168,24 @@ func main() {
	logsDir = filepath.Join(config.DistDir(), "logs")		logsDir = filepath.Join(config.DistDir(), "logs")
	}		}

			buildErrorFile := filepath.Join(logsDir, c.logsPrefix+"build_error")
			rbeMetricsFile := filepath.Join(logsDir, c.logsPrefix+"rbe_metrics.pb")
			soongMetricsFile := filepath.Join(logsDir, c.logsPrefix+"soong_metrics")
			defer build.UploadMetrics(buildCtx, config, buildStartedMilli, buildErrorFile, rbeMetricsFile, soongMetricsFile)

	os.MkdirAll(logsDir, 0777)		os.MkdirAll(logsDir, 0777)
	log.SetOutput(filepath.Join(logsDir, c.logsPrefix+"soong.log"))		log.SetOutput(filepath.Join(logsDir, c.logsPrefix+"soong.log"))
	trace.SetOutput(filepath.Join(logsDir, c.logsPrefix+"build.trace"))		trace.SetOutput(filepath.Join(logsDir, c.logsPrefix+"build.trace"))
	stat.AddOutput(status.NewVerboseLog(log, filepath.Join(logsDir, c.logsPrefix+"verbose.log")))		stat.AddOutput(status.NewVerboseLog(log, filepath.Join(logsDir, c.logsPrefix+"verbose.log")))
	stat.AddOutput(status.NewErrorLog(log, filepath.Join(logsDir, c.logsPrefix+"error.log")))		stat.AddOutput(status.NewErrorLog(log, filepath.Join(logsDir, c.logsPrefix+"error.log")))
	stat.AddOutput(status.NewProtoErrorLog(log, filepath.Join(logsDir, c.logsPrefix+"build_error")))		stat.AddOutput(status.NewProtoErrorLog(log, buildErrorFile))
	stat.AddOutput(status.NewCriticalPath(log))		stat.AddOutput(status.NewCriticalPath(log))

	buildCtx.Verbosef("Detected %.3v GB total RAM", float32(config.TotalRAM())/(102410241024))		buildCtx.Verbosef("Detected %.3v GB total RAM", float32(config.TotalRAM())/(102410241024))
	buildCtx.Verbosef("Parallelism (local/remote/highmem): %v/%v/%v",		buildCtx.Verbosef("Parallelism (local/remote/highmem): %v/%v/%v",
	config.Parallel(), config.RemoteParallel(), config.HighmemParallel())		config.Parallel(), config.RemoteParallel(), config.HighmemParallel())

	defer met.Dump(filepath.Join(logsDir, c.logsPrefix+"soong_metrics"))		defer met.Dump(soongMetricsFile)

	if start, ok := os.LookupEnv("TRACE_BEGIN_SOONG"); ok {		if start, ok := os.LookupEnv("TRACE_BEGIN_SOONG"); ok {
	if !strings.HasSuffix(start, "N") {		if !strings.HasSuffix(start, "N") {

ui/build/Android.bp

+2 −0

Original line number	Original line	Diff line number	Diff line
	@@ -56,12 +56,14 @@ bootstrap_go_package {
	"signal.go",		"signal.go",
	"soong.go",		"soong.go",
	"test_build.go",		"test_build.go",
			"upload.go",
	"util.go",		"util.go",
	],		],
	testSrcs: [		testSrcs: [
	"cleanbuild_test.go",		"cleanbuild_test.go",
	"config_test.go",		"config_test.go",
	"environment_test.go",		"environment_test.go",
			"upload_test.go",
	"util_test.go",		"util_test.go",
	"proc_sync_test.go",		"proc_sync_test.go",
	],		],

ui/build/config.go

+11 −0

Original line number	Original line	Diff line number	Diff line
	@@ -962,3 +962,14 @@ func (c *configImpl) SetPdkBuild(pdk bool) {
	func (c *configImpl) IsPdkBuild() bool {		func (c *configImpl) IsPdkBuild() bool {
	return c.pdkBuild		return c.pdkBuild
	}		}

			func (c *configImpl) BuildDateTime() string {
			return c.buildDateTime
			}

			func (c *configImpl) MetricsUploaderApp() string {
			if p, ok := c.environ.Get("ANDROID_ENABLE_METRICS_UPLOAD"); ok {
			return p
			}
			return ""
			}

ui/build/config_test.go

+2 −0

Original line number	Original line	Diff line number	Diff line
	@@ -26,6 +26,7 @@ import (
	"testing"		"testing"

	"android/soong/ui/logger"		"android/soong/ui/logger"
			"android/soong/ui/status"
	)		)

	func testContext() Context {		func testContext() Context {
	@@ -33,6 +34,7 @@ func testContext() Context {
	Context: context.Background(),		Context: context.Background(),
	Logger: logger.New(&bytes.Buffer{}),		Logger: logger.New(&bytes.Buffer{}),
	Writer: &bytes.Buffer{},		Writer: &bytes.Buffer{},
			Status: &status.Status{},
	}}		}}
	}		}

ui/build/upload.go

0 → 100644

+80 −0

Original line number	Original line	Diff line number	Diff line
			// Copyright 2020 Google Inc. All rights reserved.
			//
			// Licensed under the Apache License, Version 2.0 (the "License");
			// you may not use this file except in compliance with the License.
			// You may obtain a copy of the License at
			//
			// http://www.apache.org/licenses/LICENSE-2.0
			//
			// Unless required by applicable law or agreed to in writing, software
			// distributed under the License is distributed on an "AS IS" BASIS,
			// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
			// See the License for the specific language governing permissions and
			// limitations under the License.

			package build

			// This file contains the functionality to upload data from one location to
			// another.

			import (
			"io/ioutil"
			"os"
			"path/filepath"
			"time"

			"github.com/golang/protobuf/proto"

			upload_proto "android/soong/ui/metrics/upload_proto"
			)

			const (
			uploadPbFilename = ".uploader.pb"
			)

			// UploadMetrics uploads a set of metrics files to a server for analysis. An
			// uploader full path is required to be specified in order to upload the set
			// of metrics files. This is accomplished by defining the ANDROID_ENABLE_METRICS_UPLOAD
			// environment variable.
			func UploadMetrics(ctx Context, config Config, buildStartedMilli int64, files ...string) {
			uploader := config.MetricsUploaderApp()
			// No metrics to upload if the path to the uploader was not specified.
			if uploader == "" {
			return
			}

			// Some files may not exist. For example, build errors protobuf file
			// may not exist since the build was successful.
			var metricsFiles []string
			for _, f := range files {
			if _, err := os.Stat(f); err == nil {
			metricsFiles = append(metricsFiles, f)
			}
			}

			if len(metricsFiles) == 0 {
			return
			}

			// For platform builds, the branch and target name is hardcoded to specific
			// values for later extraction of the metrics in the data metrics pipeline.
			data, err := proto.Marshal(&upload_proto.Upload{
			CreationTimestampMs: proto.Uint64(uint64(buildStartedMilli)),
			CompletionTimestampMs: proto.Uint64(uint64(time.Now().UnixNano() / int64(time.Millisecond))),
			BranchName: proto.String("developer-metrics"),
			TargetName: proto.String("platform-build-systems-metrics"),
			MetricsFiles: metricsFiles,
			})
			if err != nil {
			ctx.Fatalf("failed to marshal metrics upload proto buffer message: %v\n", err)
			}

			pbFile := filepath.Join(config.OutDir(), uploadPbFilename)
			if err := ioutil.WriteFile(pbFile, data, 0644); err != nil {
			ctx.Fatalf("failed to write the marshaled metrics upload protobuf to %q: %v\n", pbFile, err)
			}
			// Remove the upload file as it's not longer needed after it has been processed by the uploader.
			defer os.Remove(pbFile)

			Command(ctx, config, "upload metrics", uploader, "--upload-metrics", pbFile).RunAndStreamOrFatal()
			}