Merge "Code metadata integration with Generator tool" into main (58b049f7) · Commits · e / os / android_build

tools/metadata/Android.bp

+2 −0

Original line number	Diff line number	Diff line
		@@ -6,6 +6,8 @@ blueprint_go_binary {
		name: "metadata",
		deps: [
		"soong-testing-test_spec_proto",
		"soong-testing-code_metadata_proto",
		"soong-testing-code_metadata_internal_proto",
		"golang-protobuf-proto",
		],
		srcs: [

tools/metadata/generator.go

+138 −14

Original line number	Diff line number	Diff line
		@@ -10,6 +10,8 @@ import (
		"strings"
		"sync"

		"android/soong/testing/code_metadata_internal_proto"
		"android/soong/testing/code_metadata_proto"
		"android/soong/testing/test_spec_proto"
		"google.golang.org/protobuf/proto"
		)
		@@ -23,6 +25,13 @@ func (kl keyToLocksMap) GetLockForKey(key string) sync.Mutex {
		return mutex.(*sync.Mutex)
		}

		// Define a struct to hold the combination of team ID and multi-ownership flag for validation
		type sourceFileAttributes struct {
		TeamID string
		MultiOwnership bool
		Path string
		}

		func getSortedKeys(syncMap *sync.Map) []string {
		var allKeys []string
		syncMap.Range(
		@@ -36,14 +45,9 @@ func getSortedKeys(syncMap *sync.Map) []string {
		return allKeys
		}

		func writeOutput(
		outputFile string,
		allMetadata []*test_spec_proto.TestSpec_OwnershipMetadata,
		) {
		testSpec := &test_spec_proto.TestSpec{
		OwnershipMetadataList: allMetadata,
		}
		data, err := proto.Marshal(testSpec)
		// writeProtoToFile marshals a protobuf message and writes it to a file
		func writeProtoToFile(outputFile string, message proto.Message) {
		data, err := proto.Marshal(message)
		if err != nil {
		log.Fatal(err)
		}
		@@ -141,10 +145,86 @@ func processTestSpecProtobuf(
		}
		}

		// processCodeMetadataProtobuf processes CodeMetadata protobuf files
		func processCodeMetadataProtobuf(
		filePath string, ownershipMetadataMap sync.Map, sourceFileMetadataMap sync.Map, keyLocks *keyToLocksMap,
		errCh chan error, wg *sync.WaitGroup,
		) {
		defer wg.Done()

		fileContent := strings.TrimRight(readFileToString(filePath), "\n")
		internalCodeData := code_metadata_internal_proto.CodeMetadataInternal{}
		err := proto.Unmarshal([]byte(fileContent), &internalCodeData)
		if err != nil {
		errCh <- err
		return
		}

		// Process each TargetOwnership entry
		for _, internalMetadata := range internalCodeData.GetTargetOwnershipList() {
		key := internalMetadata.GetTargetName()
		lock := keyLocks.GetLockForKey(key)
		lock.Lock()

		for _, srcFile := range internalMetadata.GetSourceFiles() {
		srcFileKey := srcFile
		srcFileLock := keyLocks.GetLockForKey(srcFileKey)
		srcFileLock.Lock()
		attributes := sourceFileAttributes{
		TeamID: internalMetadata.GetTrendyTeamId(),
		MultiOwnership: internalMetadata.GetMultiOwnership(),
		Path: internalMetadata.GetPath(),
		}

		existingAttributes, exists := sourceFileMetadataMap.Load(srcFileKey)
		if exists {
		existing := existingAttributes.(sourceFileAttributes)
		if attributes.TeamID != existing.TeamID && (!attributes.MultiOwnership \|\| !existing.MultiOwnership) {
		errCh <- fmt.Errorf(
		"Conflict found for source file %s covered at %s with team ID: %s. Existing team ID: %s and path: %s."+
		" If multi-ownership is required, multiOwnership should be set to true in all test_spec modules using this target. "+
		"Multiple-ownership in general is discouraged though as it make infrastructure around android relying on this information pick up a random value when it needs only one.",
		srcFile, internalMetadata.GetPath(), attributes.TeamID, existing.TeamID, existing.Path,
		)
		srcFileLock.Unlock()
		lock.Unlock()
		return
		}
		} else {
		// Store the metadata if no conflict
		sourceFileMetadataMap.Store(srcFileKey, attributes)
		}
		srcFileLock.Unlock()
		}

		value, loaded := ownershipMetadataMap.LoadOrStore(
		key, []*code_metadata_internal_proto.CodeMetadataInternal_TargetOwnership{internalMetadata},
		)
		if loaded {
		existingMetadata := value.([]*code_metadata_internal_proto.CodeMetadataInternal_TargetOwnership)
		isDuplicate := false
		for _, existing := range existingMetadata {
		if internalMetadata.GetTrendyTeamId() == existing.GetTrendyTeamId() && internalMetadata.GetPath() == existing.GetPath() {
		isDuplicate = true
		break
		}
		}
		if !isDuplicate {
		existingMetadata = append(existingMetadata, internalMetadata)
		ownershipMetadataMap.Store(key, existingMetadata)
		}
		}

		lock.Unlock()
		}
		}

		func main() {
		inputFile := flag.String("inputFile", "", "Input file path")
		outputFile := flag.String("outputFile", "", "Output file path")
		rule := flag.String("rule", "", "Metadata rule (Hint: test_spec or code_metadata)")
		rule := flag.String(
		"rule", "", "Metadata rule (Hint: test_spec or code_metadata)",
		)
		flag.Parse()

		if inputFile == "" \|\| outputFile == "" \|\| *rule == "" {
		@@ -167,7 +247,9 @@ func main() {
		case "test_spec":
		for _, filePath := range filePaths {
		wg.Add(1)
		go processTestSpecProtobuf(filePath, ownershipMetadataMap, keyLocks, errCh, &wg)
		go processTestSpecProtobuf(
		filePath, ownershipMetadataMap, keyLocks, errCh, &wg,
		)
		}

		wg.Wait()
		@@ -186,9 +268,51 @@ func main() {
		allMetadata = append(allMetadata, metadataList...)
		}

		writeOutput(*outputFile, allMetadata)
		testSpec := &test_spec_proto.TestSpec{
		OwnershipMetadataList: allMetadata,
		}
		writeProtoToFile(*outputFile, testSpec)
		break
		case "code_metadata":
		sourceFileMetadataMap := &sync.Map{}
		for _, filePath := range filePaths {
		wg.Add(1)
		go processCodeMetadataProtobuf(
		filePath, ownershipMetadataMap, sourceFileMetadataMap, keyLocks, errCh, &wg,
		)
		}

		wg.Wait()
		close(errCh)

		for err := range errCh {
		log.Fatal(err)
		}

		sortedKeys := getSortedKeys(ownershipMetadataMap)
		allMetadata := make([]*code_metadata_proto.CodeMetadata_TargetOwnership, 0)
		for _, key := range sortedKeys {
		value, _ := ownershipMetadataMap.Load(key)
		metadata := value.([]*code_metadata_internal_proto.CodeMetadataInternal_TargetOwnership)
		for _, m := range metadata {
		targetName := m.GetTargetName()
		path := m.GetPath()
		trendyTeamId := m.GetTrendyTeamId()

		allMetadata = append(allMetadata, &code_metadata_proto.CodeMetadata_TargetOwnership{
		TargetName: &targetName,
		Path: &path,
		TrendyTeamId: &trendyTeamId,
		SourceFiles: m.GetSourceFiles(),
		})
		}
		}

		finalMetadata := &code_metadata_proto.CodeMetadata{
		TargetOwnershipList: allMetadata,
		}
		writeProtoToFile(*outputFile, finalMetadata)
		break
		default:
		log.Fatalf("No specific processing implemented for rule '%s'.\n", *rule)
		}

tools/metadata/go.work

+2 −1

Original line number	Diff line number	Diff line
		@@ -4,7 +4,8 @@ use (
		.
		../../../../external/golang-protobuf
		../../../soong/testing/test_spec_proto

		../../../soong/testing/code_metadata_proto
		../../../soong/testing/code_metadata_proto_internal
		)

		replace google.golang.org/protobuf v0.0.0 => ../../../../external/golang-protobuf

tools/metadata/testdata/expectedCodeMetadataOutput.txt

0 → 100644

+7 −0

Original line number	Diff line number	Diff line


		bar
		Android.bp12346"b.java

		foo
		Android.bp12345"a.java
		No newline at end of file

tools/metadata/testdata/file5.txt

0 → 100644

+4 −0

Original line number	Diff line number	Diff line


		foo
		Android.bp12345"a.java