Merge "Upload build metrics after a build is completed." (039b04f4) · Commits · e / os / android_build_soong

cmd/soong_ui/main.go

+9 −2

Original line number	Diff line number	Diff line
		@@ -117,6 +117,8 @@ func inList(s string, list []string) bool {
		// Command is the type of soong_ui execution. Only one type of
		// execution is specified. The args are specific to the command.
		func main() {
		buildStartedMilli := time.Now().UnixNano() / int64(time.Millisecond)

		c, args := getCommand(os.Args)
		if c == nil {
		fmt.Fprintf(os.Stderr, "The `soong` native UI is not yet available.\n")
		@@ -166,12 +168,17 @@ func main() {
		logsDir = filepath.Join(config.DistDir(), "logs")
		}

		buildErrorFile := filepath.Join(logsDir, c.logsPrefix+"build_error")
		rbeMetricsFile := filepath.Join(logsDir, c.logsPrefix+"rbe_metrics.pb")
		soongMetricsFile := filepath.Join(logsDir, c.logsPrefix+"soong_metrics")
		defer build.UploadMetrics(buildCtx, config, buildStartedMilli, buildErrorFile, rbeMetricsFile, soongMetricsFile)

		os.MkdirAll(logsDir, 0777)
		log.SetOutput(filepath.Join(logsDir, c.logsPrefix+"soong.log"))
		trace.SetOutput(filepath.Join(logsDir, c.logsPrefix+"build.trace"))
		stat.AddOutput(status.NewVerboseLog(log, filepath.Join(logsDir, c.logsPrefix+"verbose.log")))
		stat.AddOutput(status.NewErrorLog(log, filepath.Join(logsDir, c.logsPrefix+"error.log")))
		stat.AddOutput(status.NewProtoErrorLog(log, filepath.Join(logsDir, c.logsPrefix+"build_error")))
		stat.AddOutput(status.NewProtoErrorLog(log, buildErrorFile))
		stat.AddOutput(status.NewCriticalPath(log))
		stat.AddOutput(status.NewBuildProgressLog(log, filepath.Join(logsDir, c.logsPrefix+"build_progress.pb")))

		@@ -179,7 +186,7 @@ func main() {
		buildCtx.Verbosef("Parallelism (local/remote/highmem): %v/%v/%v",
		config.Parallel(), config.RemoteParallel(), config.HighmemParallel())

		defer met.Dump(filepath.Join(logsDir, c.logsPrefix+"soong_metrics"))
		defer met.Dump(soongMetricsFile)

		if start, ok := os.LookupEnv("TRACE_BEGIN_SOONG"); ok {
		if !strings.HasSuffix(start, "N") {

ui/build/Android.bp

+2 −0

Original line number	Diff line number	Diff line
		@@ -56,12 +56,14 @@ bootstrap_go_package {
		"signal.go",
		"soong.go",
		"test_build.go",
		"upload.go",
		"util.go",
		],
		testSrcs: [
		"cleanbuild_test.go",
		"config_test.go",
		"environment_test.go",
		"upload_test.go",
		"util_test.go",
		"proc_sync_test.go",
		],

ui/build/config.go

+11 −0

Original line number	Diff line number	Diff line
		@@ -961,3 +961,14 @@ func (c *configImpl) SetPdkBuild(pdk bool) {
		func (c *configImpl) IsPdkBuild() bool {
		return c.pdkBuild
		}

		func (c *configImpl) BuildDateTime() string {
		return c.buildDateTime
		}

		func (c *configImpl) MetricsUploaderApp() string {
		if p, ok := c.environ.Get("ANDROID_ENABLE_METRICS_UPLOAD"); ok {
		return p
		}
		return ""
		}

ui/build/config_test.go

+2 −0

Original line number	Diff line number	Diff line
		@@ -26,6 +26,7 @@ import (
		"testing"

		"android/soong/ui/logger"
		"android/soong/ui/status"
		)

		func testContext() Context {
		@@ -33,6 +34,7 @@ func testContext() Context {
		Context: context.Background(),
		Logger: logger.New(&bytes.Buffer{}),
		Writer: &bytes.Buffer{},
		Status: &status.Status{},
		}}
		}

ui/build/upload.go

0 → 100644

+80 −0

Original line number	Diff line number	Diff line
		// Copyright 2020 Google Inc. All rights reserved.
		//
		// Licensed under the Apache License, Version 2.0 (the "License");
		// you may not use this file except in compliance with the License.
		// You may obtain a copy of the License at
		//
		// http://www.apache.org/licenses/LICENSE-2.0
		//
		// Unless required by applicable law or agreed to in writing, software
		// distributed under the License is distributed on an "AS IS" BASIS,
		// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
		// See the License for the specific language governing permissions and
		// limitations under the License.

		package build

		// This file contains the functionality to upload data from one location to
		// another.

		import (
		"io/ioutil"
		"os"
		"path/filepath"
		"time"

		"github.com/golang/protobuf/proto"

		upload_proto "android/soong/ui/metrics/upload_proto"
		)

		const (
		uploadPbFilename = ".uploader.pb"
		)

		// UploadMetrics uploads a set of metrics files to a server for analysis. An
		// uploader full path is required to be specified in order to upload the set
		// of metrics files. This is accomplished by defining the ANDROID_ENABLE_METRICS_UPLOAD
		// environment variable.
		func UploadMetrics(ctx Context, config Config, buildStartedMilli int64, files ...string) {
		uploader := config.MetricsUploaderApp()
		// No metrics to upload if the path to the uploader was not specified.
		if uploader == "" {
		return
		}

		// Some files may not exist. For example, build errors protobuf file
		// may not exist since the build was successful.
		var metricsFiles []string
		for _, f := range files {
		if _, err := os.Stat(f); err == nil {
		metricsFiles = append(metricsFiles, f)
		}
		}

		if len(metricsFiles) == 0 {
		return
		}

		// For platform builds, the branch and target name is hardcoded to specific
		// values for later extraction of the metrics in the data metrics pipeline.
		data, err := proto.Marshal(&upload_proto.Upload{
		CreationTimestampMs: proto.Uint64(uint64(buildStartedMilli)),
		CompletionTimestampMs: proto.Uint64(uint64(time.Now().UnixNano() / int64(time.Millisecond))),
		BranchName: proto.String("developer-metrics"),
		TargetName: proto.String("platform-build-systems-metrics"),
		MetricsFiles: metricsFiles,
		})
		if err != nil {
		ctx.Fatalf("failed to marshal metrics upload proto buffer message: %v\n", err)
		}

		pbFile := filepath.Join(config.OutDir(), uploadPbFilename)
		if err := ioutil.WriteFile(pbFile, data, 0644); err != nil {
		ctx.Fatalf("failed to write the marshaled metrics upload protobuf to %q: %v\n", pbFile, err)
		}
		// Remove the upload file as it's not longer needed after it has been processed by the uploader.
		defer os.Remove(pbFile)

		Command(ctx, config, "upload metrics", uploader, "--upload-metrics", pbFile).RunAndStreamOrFatal()
		}