Merge "Add performance counter metrics to build.trace.gz" into main (66d6409e) · Commits · e / os / android_build_soong

android/metrics.go

+134 −20

Original line number	Diff line number	Diff line
		@@ -15,9 +15,13 @@
		package android

		import (
		"bytes"
		"io/ioutil"
		"os"
		"runtime"
		"sort"
		"strconv"
		"time"

		"github.com/google/blueprint/metrics"
		"google.golang.org/protobuf/proto"
		@@ -27,18 +31,21 @@ import (

		var soongMetricsOnceKey = NewOnceKey("soong metrics")

		type SoongMetrics struct {
		Modules int
		Variants int
		type soongMetrics struct {
		modules int
		variants int
		perfCollector perfCollector
		}

		func readSoongMetrics(config Config) (SoongMetrics, bool) {
		soongMetrics, ok := config.Peek(soongMetricsOnceKey)
		if ok {
		return soongMetrics.(SoongMetrics), true
		} else {
		return SoongMetrics{}, false
		type perfCollector struct {
		events []*soong_metrics_proto.PerfCounters
		stop chan<- bool
		}

		func getSoongMetrics(config Config) *soongMetrics {
		return config.Once(soongMetricsOnceKey, func() interface{} {
		return &soongMetrics{}
		}).(*soongMetrics)
		}

		func init() {
		@@ -50,27 +57,27 @@ func soongMetricsSingletonFactory() Singleton { return soongMetricsSingleton{} }
		type soongMetricsSingleton struct{}

		func (soongMetricsSingleton) GenerateBuildActions(ctx SingletonContext) {
		metrics := SoongMetrics{}
		metrics := getSoongMetrics(ctx.Config())
		ctx.VisitAllModules(func(m Module) {
		if ctx.PrimaryModule(m) == m {
		metrics.Modules++
		metrics.modules++
		}
		metrics.Variants++
		})
		ctx.Config().Once(soongMetricsOnceKey, func() interface{} {
		return metrics
		metrics.variants++
		})
		}

		func collectMetrics(config Config, eventHandler metrics.EventHandler) soong_metrics_proto.SoongBuildMetrics {
		metrics := &soong_metrics_proto.SoongBuildMetrics{}

		soongMetrics, ok := readSoongMetrics(config)
		if ok {
		metrics.Modules = proto.Uint32(uint32(soongMetrics.Modules))
		metrics.Variants = proto.Uint32(uint32(soongMetrics.Variants))
		soongMetrics := getSoongMetrics(config)
		if soongMetrics.modules > 0 {
		metrics.Modules = proto.Uint32(uint32(soongMetrics.modules))
		metrics.Variants = proto.Uint32(uint32(soongMetrics.variants))
		}

		soongMetrics.perfCollector.stop <- true
		metrics.PerfCounters = soongMetrics.perfCollector.events

		memStats := runtime.MemStats{}
		runtime.ReadMemStats(&memStats)
		metrics.MaxHeapSize = proto.Uint64(memStats.HeapSys)
		@@ -107,6 +114,113 @@ func collectMetrics(config Config, eventHandler metrics.EventHandler) soong_me
		return metrics
		}

		func StartBackgroundMetrics(config Config) {
		perfCollector := &getSoongMetrics(config).perfCollector
		stop := make(chan bool)
		perfCollector.stop = stop

		previousTime := time.Now()
		previousCpuTime := readCpuTime()

		ticker := time.NewTicker(time.Second)

		go func() {
		for {
		select {
		case <-stop:
		ticker.Stop()
		return
		case <-ticker.C:
		// carry on
		}

		currentTime := time.Now()

		var memStats runtime.MemStats
		runtime.ReadMemStats(&memStats)

		currentCpuTime := readCpuTime()

		interval := currentTime.Sub(previousTime)
		intervalCpuTime := currentCpuTime - previousCpuTime
		intervalCpuPercent := intervalCpuTime * 100 / interval

		// heapAlloc is the memory that has been allocated on the heap but not yet GC'd. It may be referenced,
		// or unrefenced but not yet GC'd.
		heapAlloc := memStats.HeapAlloc
		// heapUnused is the memory that was previously used by the heap, but is currently not used. It does not
		// count memory that was used and then returned to the OS.
		heapUnused := memStats.HeapIdle - memStats.HeapReleased
		// heapOverhead is the memory used by the allocator and GC
		heapOverhead := memStats.MSpanSys + memStats.MCacheSys + memStats.GCSys
		// otherMem is the memory used outside of the heap.
		otherMem := memStats.Sys - memStats.HeapSys - heapOverhead

		perfCollector.events = append(perfCollector.events, &soong_metrics_proto.PerfCounters{
		Time: proto.Uint64(uint64(currentTime.UnixNano())),
		Groups: []*soong_metrics_proto.PerfCounterGroup{
		{
		Name: proto.String("cpu"),
		Counters: []*soong_metrics_proto.PerfCounter{
		{Name: proto.String("cpu_percent"), Value: proto.Int64(int64(intervalCpuPercent))},
		},
		}, {
		Name: proto.String("memory"),
		Counters: []*soong_metrics_proto.PerfCounter{
		{Name: proto.String("heap_alloc"), Value: proto.Int64(int64(heapAlloc))},
		{Name: proto.String("heap_unused"), Value: proto.Int64(int64(heapUnused))},
		{Name: proto.String("heap_overhead"), Value: proto.Int64(int64(heapOverhead))},
		{Name: proto.String("other"), Value: proto.Int64(int64(otherMem))},
		},
		},
		},
		})

		previousTime = currentTime
		previousCpuTime = currentCpuTime
		}
		}()
		}

		func readCpuTime() time.Duration {
		if runtime.GOOS != "linux" {
		return 0
		}

		stat, err := os.ReadFile("/proc/self/stat")
		if err != nil {
		return 0
		}

		endOfComm := bytes.LastIndexByte(stat, ')')
		if endOfComm < 0 \|\| endOfComm > len(stat)-2 {
		return 0
		}

		stat = stat[endOfComm+2:]

		statFields := bytes.Split(stat, []byte{' '})
		// This should come from sysconf(_SC_CLK_TCK), but there's no way to call that from Go. Assume it's 100,
		// which is the value for all platforms we support.
		const HZ = 100
		const MS_PER_HZ = 1e3 / HZ * time.Millisecond

		const STAT_UTIME_FIELD = 14 - 2
		const STAT_STIME_FIELD = 15 - 2
		if len(statFields) < STAT_STIME_FIELD {
		return 0
		}
		userCpuTicks, err := strconv.ParseUint(string(statFields[STAT_UTIME_FIELD]), 10, 64)
		if err != nil {
		return 0
		}
		kernelCpuTicks, _ := strconv.ParseUint(string(statFields[STAT_STIME_FIELD]), 10, 64)
		if err != nil {
		return 0
		}
		return time.Duration(userCpuTicks+kernelCpuTicks) * MS_PER_HZ
		}

		func WriteMetrics(config Config, eventHandler *metrics.EventHandler, metricsFile string) error {
		metrics := collectMetrics(config, eventHandler)

cmd/soong_build/main.go

+1 −0

Original line number	Diff line number	Diff line
		@@ -422,6 +422,7 @@ func main() {
		metricsDir := availableEnv["LOG_DIR"]

		ctx := newContext(configuration)
		android.StartBackgroundMetrics(configuration)

		var finalOutputFile string

ui/build/soong.go

+14 −0

Original line number	Diff line number	Diff line
		@@ -15,6 +15,7 @@
		package build

		import (
		"android/soong/ui/tracer"
		"fmt"
		"io/fs"
		"os"
		@@ -774,6 +775,19 @@ func loadSoongBuildMetrics(ctx Context, config Config, oldTimestamp time.Time) {
		ctx.Tracer.Complete(desc, ctx.Thread,
		event.GetStartTime(), event.GetStartTime()+event.GetRealTime())
		}
		for _, event := range soongBuildMetrics.PerfCounters {
		timestamp := event.GetTime()
		for _, group := range event.Groups {
		counters := make([]tracer.Counter, 0, len(group.Counters))
		for _, counter := range group.Counters {
		counters = append(counters, tracer.Counter{
		Name: counter.GetName(),
		Value: counter.GetValue(),
		})
		}
		ctx.Tracer.CountersAtTime(group.GetName(), ctx.Thread, timestamp, counters)
		}
		}
		}

		func runMicrofactory(ctx Context, config Config, name string, pkg string, mapping map[string]string) {

ui/metrics/metrics_proto/metrics.pb.go

+431 −188

File changed.

Preview size limit exceeded, changes collapsed.

ui/metrics/metrics_proto/metrics.proto

+27 −0

Original line number	Diff line number	Diff line
		@@ -213,6 +213,30 @@ message PerfInfo {
		optional string error_message = 8;
		}

		message PerfCounters {
		// The timestamp of these counters in nanoseconds.
		optional uint64 time = 1;

		// A list of counter names and values.
		repeated PerfCounterGroup groups = 2;
		}

		message PerfCounterGroup {
		// The name of this counter group (e.g. "cpu" or "memory")
		optional string name = 1;

		// The counters in this group
		repeated PerfCounter counters = 2;
		}

		message PerfCounter {
		// The name of this counter.
		optional string name = 1;

		// The value of this counter.
		optional int64 value = 2;
		}

		message ProcessResourceInfo {
		// The name of the process for identification.
		optional string name = 1;
		@@ -295,6 +319,9 @@ message SoongBuildMetrics {

		// Mixed Builds information
		optional MixedBuildsInfo mixed_builds_info = 7;

		// Performance during for soong_build execution.
		repeated PerfCounters perf_counters = 8;
		}

		message ExpConfigFetcher {