Merge "find_input_delta: Add jar inspection, metrics generation" into main (1caa78e9) · Commits · e / os / android_build_soong

cmd/find_input_delta/find_input_delta/main.go

+7 −0

Original line number	Diff line number	Diff line
		@@ -85,4 +85,11 @@ func main() {
		if err = file_list.Format(os.Stdout, template); err != nil {
		panic(err)
		}

		metrics_file := os.Getenv("SOONG_METRICS_AGGREGATION_FILE")
		if metrics_file != "" {
		if err = file_list.SendMetrics(metrics_file); err != nil {
		panic(err)
		}
		}
		}

cmd/find_input_delta/find_input_delta_lib/Android.bp

+1 −0

Original line number	Diff line number	Diff line
		@@ -25,6 +25,7 @@ bootstrap_go_package {
		"golang-protobuf-runtime-protoimpl",
		"soong-cmd-find_input_delta-proto",
		"soong-cmd-find_input_delta-proto_internal",
		"android-archive-zip",
		"blueprint-pathtools",
		],
		srcs: [

cmd/find_input_delta/find_input_delta_lib/file_list.go

+134 −9

Original line number	Diff line number	Diff line
		@@ -15,10 +15,15 @@
		package find_input_delta_lib

		import (
		"fmt"
		"io"
		"os"
		"path/filepath"
		"slices"
		"text/template"

		fid_exp "android/soong/cmd/find_input_delta/find_input_delta_proto"
		"google.golang.org/protobuf/encoding/protowire"
		"google.golang.org/protobuf/proto"
		)

		@@ -47,28 +52,148 @@ type FileList struct {

		// The modified files
		Changes []FileList

		// Map of file_extension:counts
		ExtCountMap map[string]*FileCounts

		// Total number of added/changed/deleted files.
		TotalDelta uint32
		}

		// The maximum number of files that will be recorded by name.
		var MaxFilesRecorded uint32 = 50

		type FileCounts struct {
		Additions uint32
		Deletions uint32
		Changes uint32
		}

		func FileListFactory(name string) *FileList {
		return &FileList{
		Name: name,
		ExtCountMap: make(map[string]*FileCounts),
		}
		}

		func (fl *FileList) addFile(name string) {
		fl.Additions = append(fl.Additions, name)
		fl.TotalDelta += 1
		ext := filepath.Ext(name)
		if _, ok := fl.ExtCountMap[ext]; !ok {
		fl.ExtCountMap[ext] = &FileCounts{}
		}
		fl.ExtCountMap[ext].Additions += 1
		}

		func (fl *FileList) deleteFile(name string) {
		fl.Deletions = append(fl.Deletions, name)
		fl.TotalDelta += 1
		ext := filepath.Ext(name)
		if _, ok := fl.ExtCountMap[ext]; !ok {
		fl.ExtCountMap[ext] = &FileCounts{}
		}
		fl.ExtCountMap[ext].Deletions += 1
		}

		func (fl FileList) changeFile(name string, ch FileList) {
		fl.Changes = append(fl.Changes, *ch)
		fl.TotalDelta += 1
		ext := filepath.Ext(name)
		if _, ok := fl.ExtCountMap[ext]; !ok {
		fl.ExtCountMap[ext] = &FileCounts{}
		}
		fl.ExtCountMap[ext].Changes += 1
		}

		func (fl FileList) Marshal() (*fid_exp.FileList, error) {
		func (fl FileList) ToProto() (*fid_exp.FileList, error) {
		var count uint32
		return fl.toProto(&count)
		}

		func (fl FileList) toProto(count uint32) (fid_exp.FileList, error) {
		ret := &fid_exp.FileList{
		Name: proto.String(fl.Name),
		}
		if len(fl.Additions) > 0 {
		ret.Additions = fl.Additions
		for _, a := range fl.Additions {
		if *count >= MaxFilesRecorded {
		break
		}
		ret.Additions = append(ret.Additions, a)
		*count += 1
		}
		for _, ch := range fl.Changes {
		change, err := ch.Marshal()
		if *count >= MaxFilesRecorded {
		break
		} else {
		// Pre-increment to limit what the call adds.
		*count += 1
		change, err := ch.toProto(count)
		if err != nil {
		return nil, err
		}
		ret.Changes = append(ret.Changes, change)
		}
		if len(fl.Deletions) > 0 {
		ret.Deletions = fl.Deletions
		}
		for _, d := range fl.Deletions {
		if *count >= MaxFilesRecorded {
		break
		}
		ret.Deletions = append(ret.Deletions, d)
		}
		ret.TotalDelta = proto.Uint32(*count)
		exts := []string{}
		for k := range fl.ExtCountMap {
		exts = append(exts, k)
		}
		slices.Sort(exts)
		for _, k := range exts {
		v := fl.ExtCountMap[k]
		ret.Counts = append(ret.Counts, &fid_exp.FileCount{
		Extension: proto.String(k),
		Additions: proto.Uint32(v.Additions),
		Deletions: proto.Uint32(v.Deletions),
		Modifications: proto.Uint32(v.Changes),
		})
		}
		return ret, nil
		}

		func (fl FileList) SendMetrics(path string) error {
		if path == "" {
		return fmt.Errorf("No path given")
		}
		message, err := fl.ToProto()
		if err != nil {
		return err
		}

		// Marshal the message wrapped in SoongCombinedMetrics.
		data := protowire.AppendVarint(
		[]byte{},
		protowire.EncodeTag(
		protowire.Number(fid_exp.FieldNumbers_FIELD_NUMBERS_FILE_LIST),
		protowire.BytesType))
		size := uint64(proto.Size(message))
		data = protowire.AppendVarint(data, size)
		data, err = proto.MarshalOptions{UseCachedSize: true}.MarshalAppend(data, message)
		if err != nil {
		return err
		}

		out, err := os.Create(path)
		if err != nil {
		return err
		}
		defer func() {
		if err := out.Close(); err != nil {
		fmt.Fprintf(os.Stderr, "Failed to close %s: %v\n", path, err)
		}
		}()
		_, err = out.Write(data)
		return err
		}

		func (fl FileList) Format(wr io.Writer, format string) error {
		tmpl, err := template.New("filelist").Parse(format)
		if err != nil {

cmd/find_input_delta/find_input_delta_lib/fs.go

+0 −9

Original line number	Diff line number	Diff line
		@@ -15,7 +15,6 @@
		package find_input_delta_lib

		import (
		"io"
		"io/fs"
		"os"
		)
		@@ -30,14 +29,6 @@ type fileSystem interface {
		ReadFile(path string) ([]byte, error)
		}

		type file interface {
		io.Closer
		io.Reader
		io.ReaderAt
		io.Seeker
		Stat() (os.FileInfo, error)
		}

		// osFS implements fileSystem using the local disk.
		type osFS struct{}

cmd/find_input_delta/find_input_delta_lib/internal_state.go

+31 −9

Original line number	Diff line number	Diff line
		@@ -18,9 +18,11 @@ import (
		"errors"
		"fmt"
		"io/fs"
		"path/filepath"
		"slices"

		fid_proto "android/soong/cmd/find_input_delta/find_input_delta_proto_internal"
		"android/soong/third_party/zip"
		"github.com/google/blueprint/pathtools"
		"google.golang.org/protobuf/proto"
		)
		@@ -57,6 +59,7 @@ func CreateState(inputs []string, inspect_contents bool, fsys StatReadFileFS) (*
		// If we ever have an easy hash, assign it here.
		}
		if inspect_contents {
		// NOTE: When we find it useful, we can parallelize the file inspection for speed.
		contents, err := InspectFileContents(input)
		if err != nil {
		return ret, err
		@@ -73,10 +76,31 @@ func CreateState(inputs []string, inspect_contents bool, fsys StatReadFileFS) (*
		// Inspect the file and extract the state of the elements in the archive.
		// If this is not an archive of some sort, nil is returned.
		func InspectFileContents(name string) ([]*fid_proto.PartialCompileInput, error) {
		// TODO: Actually inspect the contents.
		fmt.Printf("inspecting contents for %s\n", name)
		switch filepath.Ext(name) {
		case ".jar", ".apex", ".apk":
		return inspectZipFileContents(name)
		default:
		return nil, nil
		}
		}

		func inspectZipFileContents(name string) ([]*fid_proto.PartialCompileInput, error) {
		rc, err := zip.OpenReader(name)
		if err != nil {
		return nil, err
		}
		ret := []*fid_proto.PartialCompileInput{}
		for _, v := range rc.File {
		pci := &fid_proto.PartialCompileInput{
		Name: proto.String(v.Name),
		MtimeNsec: proto.Int64(v.ModTime().UnixNano()),
		Hash: proto.String(fmt.Sprintf("%08x", v.CRC32)),
		}
		ret = append(ret, pci)
		// We do not support nested inspection.
		}
		return ret, nil
		}

		func WriteState(s *fid_proto.PartialCompileInputs, path string) error {
		data, err := proto.Marshal(s)
		@@ -91,9 +115,7 @@ func CompareInternalState(prior, other *fid_proto.PartialCompileInputs, target s
		}

		func CompareInputFiles(prior, other []fid_proto.PartialCompileInput, name string) FileList {
		fl := &FileList{
		Name: name,
		}
		fl := FileListFactory(name)
		PriorMap := make(map[string]*fid_proto.PartialCompileInput, len(prior))
		// We know that the lists are properly sorted, so we can simply compare them.
		for _, v := range prior {
		@@ -105,17 +127,17 @@ func CompareInputFiles(prior, other []*fid_proto.PartialCompileInput, name strin
		otherMap[name] = v
		if _, ok := PriorMap[name]; !ok {
		// Added file
		fl.Additions = append(fl.Additions, name)
		fl.addFile(name)
		} else if !proto.Equal(PriorMap[name], v) {
		// Changed file
		fl.Changes = append(fl.Changes, *CompareInputFiles(PriorMap[name].GetContents(), v.GetContents(), name))
		fl.changeFile(name, CompareInputFiles(PriorMap[name].GetContents(), v.GetContents(), name))
		}
		}
		for _, v := range prior {
		name := v.GetName()
		if _, ok := otherMap[name]; !ok {
		// Deleted file
		fl.Deletions = append(fl.Deletions, name)
		fl.deleteFile(name)
		}
		}
		return fl