Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 8f986c3f authored by Aditya Choudhary's avatar Aditya Choudhary
Browse files

Code metadata integration with Generator tool 

Ignore-AOSP-First: CPing test_spec rule to udc-mainline-prod to support migration of test targets. Cherry pick of:aosp/2847033

Change-Id: Icf14b48bc777207ac7cc7a287a174c8de817339b
Merged-In: Icf14b48bc777207ac7cc7a287a174c8de817339b
parent e14f4862

tools/metadata/Android.bp

+2 −0
Original line number Diff line number Diff line
@@ -6,6 +6,8 @@ blueprint_go_binary { 
    name: "metadata",

    deps: [

            "soong-testing-test_spec_proto",

            "soong-testing-code_metadata_proto",

            "soong-testing-code_metadata_internal_proto",

            "golang-protobuf-proto",

        ],

    srcs: [

tools/metadata/generator.go

+138 −14
Original line number Diff line number Diff line
@@ -10,6 +10,8 @@ import ( 
	"strings"

	"sync"



	"android/soong/testing/code_metadata_internal_proto"

	"android/soong/testing/code_metadata_proto"

	"android/soong/testing/test_spec_proto"

	"google.golang.org/protobuf/proto"

)
@@ -23,6 +25,13 @@ func (kl *keyToLocksMap) GetLockForKey(key string) *sync.Mutex { 
	return mutex.(*sync.Mutex)

}



// Define a struct to hold the combination of team ID and multi-ownership flag for validation

type sourceFileAttributes struct {

	TeamID         string

	MultiOwnership bool

	Path           string

}



func getSortedKeys(syncMap *sync.Map) []string {

	var allKeys []string

	syncMap.Range(
@@ -36,14 +45,9 @@ func getSortedKeys(syncMap *sync.Map) []string { 
	return allKeys

}



func writeOutput(

	outputFile string,

	allMetadata []*test_spec_proto.TestSpec_OwnershipMetadata,

) {

	testSpec := &test_spec_proto.TestSpec{

		OwnershipMetadataList: allMetadata,

	}

	data, err := proto.Marshal(testSpec)

// writeProtoToFile marshals a protobuf message and writes it to a file

func writeProtoToFile(outputFile string, message proto.Message) {

	data, err := proto.Marshal(message)

	if err != nil {

		log.Fatal(err)

	}
@@ -141,10 +145,86 @@ func processTestSpecProtobuf( 
	}

}



// processCodeMetadataProtobuf processes CodeMetadata protobuf files

func processCodeMetadataProtobuf(

		filePath string, ownershipMetadataMap *sync.Map, sourceFileMetadataMap *sync.Map, keyLocks *keyToLocksMap,

		errCh chan error, wg *sync.WaitGroup,

) {

	defer wg.Done()



	fileContent := strings.TrimRight(readFileToString(filePath), "\n")

	internalCodeData := code_metadata_internal_proto.CodeMetadataInternal{}

	err := proto.Unmarshal([]byte(fileContent), &internalCodeData)

	if err != nil {

		errCh <- err

		return

	}



	// Process each TargetOwnership entry

	for _, internalMetadata := range internalCodeData.GetTargetOwnershipList() {

		key := internalMetadata.GetTargetName()

		lock := keyLocks.GetLockForKey(key)

		lock.Lock()



		for _, srcFile := range internalMetadata.GetSourceFiles() {

			srcFileKey := srcFile

			srcFileLock := keyLocks.GetLockForKey(srcFileKey)

			srcFileLock.Lock()

			attributes := sourceFileAttributes{

				TeamID:         internalMetadata.GetTrendyTeamId(),

				MultiOwnership: internalMetadata.GetMultiOwnership(),

				Path:           internalMetadata.GetPath(),

			}



			existingAttributes, exists := sourceFileMetadataMap.Load(srcFileKey)

			if exists {

				existing := existingAttributes.(sourceFileAttributes)

				if attributes.TeamID != existing.TeamID && (!attributes.MultiOwnership || !existing.MultiOwnership) {

					errCh <- fmt.Errorf(

						"Conflict found for source file %s covered at %s with team ID: %s. Existing team ID: %s and path: %s."+

								" If multi-ownership is required, multiOwnership should be set to true in all test_spec modules using this target. "+

								"Multiple-ownership in general is discouraged though as it make infrastructure around android relying on this information pick up a random value when it needs only one.",

						srcFile, internalMetadata.GetPath(), attributes.TeamID, existing.TeamID, existing.Path,

					)

					srcFileLock.Unlock()

					lock.Unlock()

					return

				}

			} else {

				// Store the metadata if no conflict

				sourceFileMetadataMap.Store(srcFileKey, attributes)

			}

			srcFileLock.Unlock()

		}



		value, loaded := ownershipMetadataMap.LoadOrStore(

			key, []*code_metadata_internal_proto.CodeMetadataInternal_TargetOwnership{internalMetadata},

		)

		if loaded {

			existingMetadata := value.([]*code_metadata_internal_proto.CodeMetadataInternal_TargetOwnership)

			isDuplicate := false

			for _, existing := range existingMetadata {

				if internalMetadata.GetTrendyTeamId() == existing.GetTrendyTeamId() && internalMetadata.GetPath() == existing.GetPath() {

					isDuplicate = true

					break

				}

			}

			if !isDuplicate {

				existingMetadata = append(existingMetadata, internalMetadata)

				ownershipMetadataMap.Store(key, existingMetadata)

			}

		}



		lock.Unlock()

	}

}



func main() {

	inputFile := flag.String("inputFile", "", "Input file path")

	outputFile := flag.String("outputFile", "", "Output file path")

	rule := flag.String("rule", "", "Metadata rule (Hint: test_spec or code_metadata)")

	rule := flag.String(

		"rule", "", "Metadata rule (Hint: test_spec or code_metadata)",

	)

	flag.Parse()



	if *inputFile == "" || *outputFile == "" || *rule == "" {
@@ -167,7 +247,9 @@ func main() { 
	case "test_spec":

		for _, filePath := range filePaths {

			wg.Add(1)

			go processTestSpecProtobuf(filePath, ownershipMetadataMap, keyLocks, errCh, &wg)

			go processTestSpecProtobuf(

				filePath, ownershipMetadataMap, keyLocks, errCh, &wg,

			)

		}



		wg.Wait()
@@ -186,9 +268,51 @@ func main() { 
			allMetadata = append(allMetadata, metadataList...)

		}



		writeOutput(*outputFile, allMetadata)

		testSpec := &test_spec_proto.TestSpec{

			OwnershipMetadataList: allMetadata,

		}

		writeProtoToFile(*outputFile, testSpec)

		break

	case "code_metadata":

		sourceFileMetadataMap := &sync.Map{}

		for _, filePath := range filePaths {

			wg.Add(1)

			go processCodeMetadataProtobuf(

				filePath, ownershipMetadataMap, sourceFileMetadataMap, keyLocks, errCh, &wg,

			)

		}



		wg.Wait()

		close(errCh)



		for err := range errCh {

			log.Fatal(err)

		}



		sortedKeys := getSortedKeys(ownershipMetadataMap)

		allMetadata := make([]*code_metadata_proto.CodeMetadata_TargetOwnership, 0)

		for _, key := range sortedKeys {

			value, _ := ownershipMetadataMap.Load(key)

			metadata := value.([]*code_metadata_internal_proto.CodeMetadataInternal_TargetOwnership)

			for _, m := range metadata {

				targetName := m.GetTargetName()

				path := m.GetPath()

				trendyTeamId := m.GetTrendyTeamId()



				allMetadata = append(allMetadata, &code_metadata_proto.CodeMetadata_TargetOwnership{

					TargetName:   &targetName,

					Path:         &path,

					TrendyTeamId: &trendyTeamId,

					SourceFiles:  m.GetSourceFiles(),

				})

			}

		}



		finalMetadata := &code_metadata_proto.CodeMetadata{

			TargetOwnershipList: allMetadata,

		}

		writeProtoToFile(*outputFile, finalMetadata)

		break

	default:

		log.Fatalf("No specific processing implemented for rule '%s'.\n", *rule)

	}

tools/metadata/go.work

+2 −1
Original line number Diff line number Diff line
@@ -4,7 +4,8 @@ use ( 
	.

	../../../../external/golang-protobuf

	../../../soong/testing/test_spec_proto



	../../../soong/testing/code_metadata_proto

	../../../soong/testing/code_metadata_proto_internal

)



replace google.golang.org/protobuf v0.0.0 => ../../../../external/golang-protobuf

tools/metadata/testdata/expectedCodeMetadataOutput.txt

0 → 100644
+7 −0
Original line number Diff line number Diff line 


 

bar

Android.bp12346"b.java

 

foo

Android.bp12345"a.java
 
 No newline at end of file

tools/metadata/testdata/file5.txt

0 → 100644
+4 −0
Original line number Diff line number Diff line 


 

foo

Android.bp12345"a.java