Merge "Export Soong modules and build a database from metadata from Make and Soong." into main (af64ca93) · Commits · e / os / android_build_soong

android/Android.bp

+1 −0

Original line number	Diff line number	Diff line
		@@ -40,6 +40,7 @@ bootstrap_go_package {
		"base_module_context.go",
		"build_prop.go",
		"buildinfo_prop.go",
		"compliance_metadata.go",
		"config.go",
		"test_config.go",
		"configurable_properties.go",

android/compliance_metadata.go

0 → 100644

+314 −0

Original line number	Diff line number	Diff line
		// Copyright 2024 Google Inc. All rights reserved.
		//
		// Licensed under the Apache License, Version 2.0 (the "License");
		// you may not use this file except in compliance with the License.
		// You may obtain a copy of the License at
		//
		// http://www.apache.org/licenses/LICENSE-2.0
		//
		// Unless required by applicable law or agreed to in writing, software
		// distributed under the License is distributed on an "AS IS" BASIS,
		// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
		// See the License for the specific language governing permissions and
		// limitations under the License.

		package android

		import (
		"bytes"
		"encoding/csv"
		"fmt"
		"slices"
		"strconv"
		"strings"

		"github.com/google/blueprint"
		)

		var (
		// Constants of property names used in compliance metadata of modules
		ComplianceMetadataProp = struct {
		NAME string
		PACKAGE string
		MODULE_TYPE string
		OS string
		ARCH string
		IS_PRIMARY_ARCH string
		VARIANT string
		IS_STATIC_LIB string
		INSTALLED_FILES string
		BUILT_FILES string
		STATIC_DEPS string
		STATIC_DEP_FILES string
		WHOLE_STATIC_DEPS string
		WHOLE_STATIC_DEP_FILES string
		LICENSES string

		// module_type=package
		PKG_DEFAULT_APPLICABLE_LICENSES string

		// module_type=license
		LIC_LICENSE_KINDS string
		LIC_LICENSE_TEXT string
		LIC_PACKAGE_NAME string

		// module_type=license_kind
		LK_CONDITIONS string
		LK_URL string
		}{
		"name",
		"package",
		"module_type",
		"os",
		"arch",
		"is_primary_arch",
		"variant",
		"is_static_lib",
		"installed_files",
		"built_files",
		"static_deps",
		"static_dep_files",
		"whole_static_deps",
		"whole_static_dep_files",
		"licenses",

		"pkg_default_applicable_licenses",

		"lic_license_kinds",
		"lic_license_text",
		"lic_package_name",

		"lk_conditions",
		"lk_url",
		}

		// A constant list of all property names in compliance metadata
		// Order of properties here is the order of columns in the exported CSV file.
		COMPLIANCE_METADATA_PROPS = []string{
		ComplianceMetadataProp.NAME,
		ComplianceMetadataProp.PACKAGE,
		ComplianceMetadataProp.MODULE_TYPE,
		ComplianceMetadataProp.OS,
		ComplianceMetadataProp.ARCH,
		ComplianceMetadataProp.VARIANT,
		ComplianceMetadataProp.IS_STATIC_LIB,
		ComplianceMetadataProp.IS_PRIMARY_ARCH,
		// Space separated installed files
		ComplianceMetadataProp.INSTALLED_FILES,
		// Space separated built files
		ComplianceMetadataProp.BUILT_FILES,
		// Space separated module names of static dependencies
		ComplianceMetadataProp.STATIC_DEPS,
		// Space separated file paths of static dependencies
		ComplianceMetadataProp.STATIC_DEP_FILES,
		// Space separated module names of whole static dependencies
		ComplianceMetadataProp.WHOLE_STATIC_DEPS,
		// Space separated file paths of whole static dependencies
		ComplianceMetadataProp.WHOLE_STATIC_DEP_FILES,
		ComplianceMetadataProp.LICENSES,
		// module_type=package
		ComplianceMetadataProp.PKG_DEFAULT_APPLICABLE_LICENSES,
		// module_type=license
		ComplianceMetadataProp.LIC_LICENSE_KINDS,
		ComplianceMetadataProp.LIC_LICENSE_TEXT, // resolve to file paths
		ComplianceMetadataProp.LIC_PACKAGE_NAME,
		// module_type=license_kind
		ComplianceMetadataProp.LK_CONDITIONS,
		ComplianceMetadataProp.LK_URL,
		}
		)

		// ComplianceMetadataInfo provides all metadata of a module, e.g. name, module type, package, license,
		// dependencies, built/installed files, etc. It is a wrapper on a map[string]string with some utility
		// methods to get/set properties' values.
		type ComplianceMetadataInfo struct {
		properties map[string]string
		}

		func NewComplianceMetadataInfo() *ComplianceMetadataInfo {
		return &ComplianceMetadataInfo{
		properties: map[string]string{},
		}
		}

		func (c *ComplianceMetadataInfo) SetStringValue(propertyName string, value string) {
		if !slices.Contains(COMPLIANCE_METADATA_PROPS, propertyName) {
		panic(fmt.Errorf("Unknown metadata property: %s.", propertyName))
		}
		c.properties[propertyName] = value
		}

		func (c *ComplianceMetadataInfo) SetListValue(propertyName string, value []string) {
		c.SetStringValue(propertyName, strings.TrimSpace(strings.Join(value, " ")))
		}

		func (c *ComplianceMetadataInfo) getStringValue(propertyName string) string {
		if !slices.Contains(COMPLIANCE_METADATA_PROPS, propertyName) {
		panic(fmt.Errorf("Unknown metadata property: %s.", propertyName))
		}
		return c.properties[propertyName]
		}

		func (c *ComplianceMetadataInfo) getAllValues() map[string]string {
		return c.properties
		}

		var (
		ComplianceMetadataProvider = blueprint.NewProvider[*ComplianceMetadataInfo]()
		)

		// buildComplianceMetadataProvider starts with the ModuleContext.ComplianceMetadataInfo() and fills in more common metadata
		// for different module types without accessing their private fields but through android.Module interface
		// and public/private fields of package android. The final metadata is stored to a module's ComplianceMetadataProvider.
		func buildComplianceMetadataProvider(ctx ModuleContext, m *ModuleBase) {
		complianceMetadataInfo := ctx.ComplianceMetadataInfo()
		complianceMetadataInfo.SetStringValue(ComplianceMetadataProp.NAME, m.Name())
		complianceMetadataInfo.SetStringValue(ComplianceMetadataProp.PACKAGE, ctx.ModuleDir())
		complianceMetadataInfo.SetStringValue(ComplianceMetadataProp.MODULE_TYPE, ctx.ModuleType())

		switch ctx.ModuleType() {
		case "license":
		licenseModule := m.module.(*licenseModule)
		complianceMetadataInfo.SetListValue(ComplianceMetadataProp.LIC_LICENSE_KINDS, licenseModule.properties.License_kinds)
		complianceMetadataInfo.SetListValue(ComplianceMetadataProp.LIC_LICENSE_TEXT, PathsForModuleSrc(ctx, licenseModule.properties.License_text).Strings())
		complianceMetadataInfo.SetStringValue(ComplianceMetadataProp.LIC_PACKAGE_NAME, String(licenseModule.properties.Package_name))
		case "license_kind":
		licenseKindModule := m.module.(*licenseKindModule)
		complianceMetadataInfo.SetListValue(ComplianceMetadataProp.LK_CONDITIONS, licenseKindModule.properties.Conditions)
		complianceMetadataInfo.SetStringValue(ComplianceMetadataProp.LK_URL, licenseKindModule.properties.Url)
		default:
		complianceMetadataInfo.SetStringValue(ComplianceMetadataProp.OS, ctx.Os().String())
		complianceMetadataInfo.SetStringValue(ComplianceMetadataProp.ARCH, ctx.Arch().String())
		complianceMetadataInfo.SetStringValue(ComplianceMetadataProp.IS_PRIMARY_ARCH, strconv.FormatBool(ctx.PrimaryArch()))
		complianceMetadataInfo.SetStringValue(ComplianceMetadataProp.VARIANT, ctx.ModuleSubDir())
		if m.primaryLicensesProperty != nil && m.primaryLicensesProperty.getName() == "licenses" {
		complianceMetadataInfo.SetListValue(ComplianceMetadataProp.LICENSES, m.primaryLicensesProperty.getStrings())
		}

		var installed InstallPaths
		installed = append(installed, m.module.FilesToInstall()...)
		installed = append(installed, m.katiInstalls.InstallPaths()...)
		installed = append(installed, m.katiSymlinks.InstallPaths()...)
		installed = append(installed, m.katiInitRcInstalls.InstallPaths()...)
		installed = append(installed, m.katiVintfInstalls.InstallPaths()...)
		complianceMetadataInfo.SetListValue(ComplianceMetadataProp.INSTALLED_FILES, FirstUniqueStrings(installed.Strings()))
		}
		ctx.setProvider(ComplianceMetadataProvider, complianceMetadataInfo)
		}

		func init() {
		RegisterComplianceMetadataSingleton(InitRegistrationContext)
		}

		func RegisterComplianceMetadataSingleton(ctx RegistrationContext) {
		ctx.RegisterParallelSingletonType("compliance_metadata_singleton", complianceMetadataSingletonFactory)
		}

		var (
		// sqlite3 command line tool
		sqlite3 = pctx.HostBinToolVariable("sqlite3", "sqlite3")

		// Command to import .csv files to sqlite3 database
		importCsv = pctx.AndroidStaticRule("importCsv",
		blueprint.RuleParams{
		Command: `rm -rf $out && ` +
		`${sqlite3} $out ".import --csv $in modules" && ` +
		`${sqlite3} $out ".import --csv ${make_metadata} make_metadata" && ` +
		`${sqlite3} $out ".import --csv ${make_modules} make_modules"`,
		CommandDeps: []string{"${sqlite3}"},
		}, "make_metadata", "make_modules")
		)

		func complianceMetadataSingletonFactory() Singleton {
		return &complianceMetadataSingleton{}
		}

		type complianceMetadataSingleton struct {
		}

		func writerToCsv(csvWriter *csv.Writer, row []string) {
		err := csvWriter.Write(row)
		if err != nil {
		panic(err)
		}
		}

		// Collect compliance metadata from all Soong modules, write to a CSV file and
		// import compliance metadata from Make and Soong to a sqlite3 database.
		func (c *complianceMetadataSingleton) GenerateBuildActions(ctx SingletonContext) {
		if !ctx.Config().HasDeviceProduct() {
		return
		}
		var buffer bytes.Buffer
		csvWriter := csv.NewWriter(&buffer)

		// Collect compliance metadata of modules in Soong and write to out/soong/compliance-metadata/<product>/soong-modules.csv file.
		columnNames := []string{"id"}
		columnNames = append(columnNames, COMPLIANCE_METADATA_PROPS...)
		writerToCsv(csvWriter, columnNames)

		rowId := -1
		ctx.VisitAllModules(func(module Module) {
		if !module.Enabled(ctx) {
		return
		}
		moduleType := ctx.ModuleType(module)
		if moduleType == "package" {
		metadataMap := map[string]string{
		ComplianceMetadataProp.NAME: ctx.ModuleName(module),
		ComplianceMetadataProp.MODULE_TYPE: ctx.ModuleType(module),
		ComplianceMetadataProp.PKG_DEFAULT_APPLICABLE_LICENSES: strings.Join(module.base().primaryLicensesProperty.getStrings(), " "),
		}
		rowId = rowId + 1
		metadata := []string{strconv.Itoa(rowId)}
		for _, propertyName := range COMPLIANCE_METADATA_PROPS {
		metadata = append(metadata, metadataMap[propertyName])
		}
		writerToCsv(csvWriter, metadata)
		return
		}
		if provider, ok := ctx.moduleProvider(module, ComplianceMetadataProvider); ok {
		metadataInfo := provider.(*ComplianceMetadataInfo)
		rowId = rowId + 1
		metadata := []string{strconv.Itoa(rowId)}
		for _, propertyName := range COMPLIANCE_METADATA_PROPS {
		metadata = append(metadata, metadataInfo.getStringValue(propertyName))
		}
		writerToCsv(csvWriter, metadata)
		return
		}
		})
		csvWriter.Flush()

		deviceProduct := ctx.Config().DeviceProduct()
		modulesCsv := PathForOutput(ctx, "compliance-metadata", deviceProduct, "soong-modules.csv")
		WriteFileRuleVerbatim(ctx, modulesCsv, buffer.String())

		// Metadata generated in Make
		makeMetadataCsv := PathForOutput(ctx, "compliance-metadata", deviceProduct, "make-metadata.csv")
		makeModulesCsv := PathForOutput(ctx, "compliance-metadata", deviceProduct, "make-modules.csv")

		// Import metadata from Make and Soong to sqlite3 database
		complianceMetadataDb := PathForOutput(ctx, "compliance-metadata", deviceProduct, "compliance-metadata.db")
		ctx.Build(pctx, BuildParams{
		Rule: importCsv,
		Input: modulesCsv,
		Implicits: []Path{
		makeMetadataCsv,
		makeModulesCsv,
		},
		Output: complianceMetadataDb,
		Args: map[string]string{
		"make_metadata": makeMetadataCsv.String(),
		"make_modules": makeModulesCsv.String(),
		},
		})

		// Phony rule "compliance-metadata.db". "m compliance-metadata.db" to create the compliance metadata database.
		ctx.Build(pctx, BuildParams{
		Rule: blueprint.Phony,
		Inputs: []Path{complianceMetadataDb},
		Output: PathForPhony(ctx, "compliance-metadata.db"),
		})

		}

android/module.go

+6 −0

Original line number	Diff line number	Diff line
		@@ -919,6 +919,10 @@ type ModuleBase struct {
		// outputFiles stores the output of a module by tag and is used to set
		// the OutputFilesProvider in GenerateBuildActions
		outputFiles OutputFilesInfo

		// complianceMetadataInfo is for different module types to dump metadata.
		// See android.ModuleContext interface.
		complianceMetadataInfo *ComplianceMetadataInfo
		}

		func (m ModuleBase) AddJSONData(d map[string]interface{}) {
		@@ -2049,6 +2053,8 @@ func (m *ModuleBase) GenerateBuildActions(blueprintCtx blueprint.ModuleContext)
		if m.outputFiles.DefaultOutputFiles != nil \|\| m.outputFiles.TaggedOutputFiles != nil {
		SetProvider(ctx, OutputFilesProvider, m.outputFiles)
		}

		buildComplianceMetadataProvider(ctx, m)
		}

		func SetJarJarPrefixHandler(handler func(ModuleContext)) {

android/module_context.go

+14 −0

Original line number	Diff line number	Diff line
		@@ -216,6 +216,11 @@ type ModuleContext interface {
		// SetOutputFiles stores the outputFiles to outputFiles property, which is used
		// to set the OutputFilesProvider later.
		SetOutputFiles(outputFiles Paths, tag string)

		// ComplianceMetadataInfo returns a ComplianceMetadataInfo instance for different module types to dump metadata,
		// which usually happens in GenerateAndroidBuildActions() of a module type.
		// See android.ModuleBase.complianceMetadataInfo
		ComplianceMetadataInfo() *ComplianceMetadataInfo
		}

		type moduleContext struct {
		@@ -729,6 +734,15 @@ func (m *moduleContext) SetOutputFiles(outputFiles Paths, tag string) {
		}
		}

		func (m moduleContext) ComplianceMetadataInfo() ComplianceMetadataInfo {
		if complianceMetadataInfo := m.module.base().complianceMetadataInfo; complianceMetadataInfo != nil {
		return complianceMetadataInfo
		}
		complianceMetadataInfo := NewComplianceMetadataInfo()
		m.module.base().complianceMetadataInfo = complianceMetadataInfo
		return complianceMetadataInfo
		}

		// Returns a list of paths expanded from globs and modules referenced using ":module" syntax. The property must
		// be tagged with `android:"path" to support automatic source module dependency resolution.
		//

cc/cc.go

+37 −0

Original line number	Diff line number	Diff line
		@@ -2121,6 +2121,43 @@ func (c *Module) GenerateAndroidBuildActions(actx android.ModuleContext) {
		}

		}

		buildComplianceMetadataInfo(ctx, c, deps)
		}

		func buildComplianceMetadataInfo(ctx ModuleContext, c *Module, deps PathDeps) {
		// Dump metadata that can not be done in android/compliance-metadata.go
		complianceMetadataInfo := ctx.ComplianceMetadataInfo()
		complianceMetadataInfo.SetStringValue(android.ComplianceMetadataProp.IS_STATIC_LIB, strconv.FormatBool(ctx.static()))
		complianceMetadataInfo.SetStringValue(android.ComplianceMetadataProp.BUILT_FILES, c.outputFile.String())

		// Static deps
		staticDeps := ctx.GetDirectDepsWithTag(StaticDepTag(false))
		staticDepNames := make([]string, 0, len(staticDeps))
		for _, dep := range staticDeps {
		staticDepNames = append(staticDepNames, dep.Name())
		}

		staticDepPaths := make([]string, 0, len(deps.StaticLibs))
		for _, dep := range deps.StaticLibs {
		staticDepPaths = append(staticDepPaths, dep.String())
		}
		complianceMetadataInfo.SetListValue(android.ComplianceMetadataProp.STATIC_DEPS, android.FirstUniqueStrings(staticDepNames))
		complianceMetadataInfo.SetListValue(android.ComplianceMetadataProp.STATIC_DEP_FILES, android.FirstUniqueStrings(staticDepPaths))

		// Whole static deps
		wholeStaticDeps := ctx.GetDirectDepsWithTag(StaticDepTag(true))
		wholeStaticDepNames := make([]string, 0, len(wholeStaticDeps))
		for _, dep := range wholeStaticDeps {
		wholeStaticDepNames = append(wholeStaticDepNames, dep.Name())
		}

		wholeStaticDepPaths := make([]string, 0, len(deps.WholeStaticLibs))
		for _, dep := range deps.WholeStaticLibs {
		wholeStaticDepPaths = append(wholeStaticDepPaths, dep.String())
		}
		complianceMetadataInfo.SetListValue(android.ComplianceMetadataProp.WHOLE_STATIC_DEPS, android.FirstUniqueStrings(wholeStaticDepNames))
		complianceMetadataInfo.SetListValue(android.ComplianceMetadataProp.WHOLE_STATIC_DEP_FILES, android.FirstUniqueStrings(wholeStaticDepPaths))
		}

		func (c *Module) maybeUnhideFromMake() {