Upload build metrics after a build is completed. am: d519a71f97 (a948f4be) · Commits · e / os / android_build_soong

cmd/soong_ui/main.go

+12 −1

Original line number	Diff line number	Diff line
		@@ -47,11 +47,17 @@ func inList(s string, list []string) bool {
		}

		func main() {
		buildStartedMilli := time.Now().UnixNano() / int64(time.Millisecond)
		var stdio terminal.StdioInterface
		stdio = terminal.StdioImpl{}
		logsPrefix := ""

		// dumpvar uses stdout, everything else should be in stderr
		if os.Args[1] == "--dumpvar-mode" \|\| os.Args[1] == "--dumpvars-mode" {
		// Any metrics files add the prefix to distinguish the type of metrics being
		// collected to further aggregate the metrics. For dump-var mode, it is usually
		// related to the execution of lunch command.
		logsPrefix = "dumpvars-"
		stdio = terminal.NewCustomStdio(os.Stdin, os.Stderr, os.Stderr)
		}

		@@ -110,13 +116,18 @@ func main() {
		logsDir = filepath.Join(config.DistDir(), "logs")
		}

		rbeMetricsFile := filepath.Join(logsDir, logsPrefix+"rbe_metrics.pb")
		soongMetricsFile := filepath.Join(logsDir, logsPrefix+"soong_metrics")
		defer build.UploadMetrics(buildCtx, config, buildStartedMilli, rbeMetricsFile, soongMetricsFile)

		os.MkdirAll(logsDir, 0777)

		log.SetOutput(filepath.Join(logsDir, "soong.log"))
		trace.SetOutput(filepath.Join(logsDir, "build.trace"))
		stat.AddOutput(status.NewVerboseLog(log, filepath.Join(logsDir, "verbose.log")))
		stat.AddOutput(status.NewErrorLog(log, filepath.Join(logsDir, "error.log")))

		defer met.Dump(filepath.Join(logsDir, "build_metrics"))
		defer met.Dump(soongMetricsFile)

		if start, ok := os.LookupEnv("TRACE_BEGIN_SOONG"); ok {
		if !strings.HasSuffix(start, "N") {

ui/build/Android.bp

+2 −0

Original line number	Diff line number	Diff line
		@@ -56,11 +56,13 @@ bootstrap_go_package {
		"signal.go",
		"soong.go",
		"test_build.go",
		"upload.go",
		"util.go",
		],
		testSrcs: [
		"config_test.go",
		"environment_test.go",
		"upload_test.go",
		"util_test.go",
		"proc_sync_test.go",
		],

ui/build/config.go

+7 −0

Original line number	Diff line number	Diff line
		@@ -712,3 +712,10 @@ func (c *configImpl) SetPdkBuild(pdk bool) {
		func (c *configImpl) IsPdkBuild() bool {
		return c.pdkBuild
		}

		func (c *configImpl) MetricsUploaderApp() string {
		if p, ok := c.environ.Get("ANDROID_ENABLE_METRICS_UPLOAD"); ok {
		return p
		}
		return ""
		}

ui/build/config_test.go

+2 −0

Original line number	Diff line number	Diff line
		@@ -22,6 +22,7 @@ import (
		"testing"

		"android/soong/ui/logger"
		"android/soong/ui/status"
		"android/soong/ui/terminal"
		)

		@@ -30,6 +31,7 @@ func testContext() Context {
		Context: context.Background(),
		Logger: logger.New(&bytes.Buffer{}),
		Writer: terminal.NewWriter(terminal.NewCustomStdio(&bytes.Buffer{}, &bytes.Buffer{}, &bytes.Buffer{})),
		Status: &status.Status{},
		}}
		}

ui/build/upload.go

0 → 100644

+80 −0

Original line number	Diff line number	Diff line
		// Copyright 2020 Google Inc. All rights reserved.
		//
		// Licensed under the Apache License, Version 2.0 (the "License");
		// you may not use this file except in compliance with the License.
		// You may obtain a copy of the License at
		//
		// http://www.apache.org/licenses/LICENSE-2.0
		//
		// Unless required by applicable law or agreed to in writing, software
		// distributed under the License is distributed on an "AS IS" BASIS,
		// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
		// See the License for the specific language governing permissions and
		// limitations under the License.

		package build

		// This file contains the functionality to upload data from one location to
		// another.

		import (
		"io/ioutil"
		"os"
		"path/filepath"
		"time"

		"github.com/golang/protobuf/proto"

		upload_proto "android/soong/ui/metrics/upload_proto"
		)

		const (
		uploadPbFilename = ".uploader.pb"
		)

		// UploadMetrics uploads a set of metrics files to a server for analysis. An
		// uploader full path is required to be specified in order to upload the set
		// of metrics files. This is accomplished by defining the ANDROID_ENABLE_METRICS_UPLOAD
		// environment variable.
		func UploadMetrics(ctx Context, config Config, buildStartedMilli int64, files ...string) {
		uploader := config.MetricsUploaderApp()
		// No metrics to upload if the path to the uploader was not specified.
		if uploader == "" {
		return
		}

		// Some files may not exist. For example, build errors protobuf file
		// may not exist since the build was successful.
		var metricsFiles []string
		for _, f := range files {
		if _, err := os.Stat(f); err == nil {
		metricsFiles = append(metricsFiles, f)
		}
		}

		if len(metricsFiles) == 0 {
		return
		}

		// For platform builds, the branch and target name is hardcoded to specific
		// values for later extraction of the metrics in the data metrics pipeline.
		data, err := proto.Marshal(&upload_proto.Upload{
		CreationTimestampMs: proto.Uint64(uint64(buildStartedMilli)),
		CompletionTimestampMs: proto.Uint64(uint64(time.Now().UnixNano() / int64(time.Millisecond))),
		BranchName: proto.String("developer-metrics"),
		TargetName: proto.String("platform-build-systems-metrics"),
		MetricsFiles: metricsFiles,
		})
		if err != nil {
		ctx.Fatalf("failed to marshal metrics upload proto buffer message: %v\n", err)
		}

		pbFile := filepath.Join(config.OutDir(), uploadPbFilename)
		if err := ioutil.WriteFile(pbFile, data, 0644); err != nil {
		ctx.Fatalf("failed to write the marshaled metrics upload protobuf to %q: %v\n", pbFile, err)
		}
		// Remove the upload file as it's not longer needed after it has been processed by the uploader.
		defer os.Remove(pbFile)

		Command(ctx, config, "upload metrics", uploader, "--upload-metrics", pbFile).RunAndPrintOrFatal()
		}