Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 242ef0c4 authored by Inseob Kim's avatar Inseob Kim
Browse files

Fix and generate vndk snapshot entirely in Soong 

- VNDK snapshot now respects stem and suffix.
- ld.config.txt is removed from snapshot as linkerconfig has become default.
- Soong builds entire snapshot, and make just calls dist-for-goals.

Bug: 142589718
Test: build and install snapshot
Test: development/vndk/snapshot/update.py with past version of snapshot
Change-Id: Id1ed658c22bb2e41c0ee50d1fe2a97924a76d7dc
parent f68f98ee

cc/cc_test.go

+1 −1
Original line number Diff line number Diff line
@@ -251,12 +251,12 @@ func checkVndkModule(t *testing.T, ctx *android.TestContext, name, subDir string 
func checkVndkSnapshot(t *testing.T, ctx *android.TestContext, name, subDir, variant string) {

	vndkSnapshot := ctx.SingletonForTests("vndk-snapshot")



	snapshotPath := filepath.Join(subDir, name+".so")

	mod := ctx.ModuleForTests(name, variant).Module().(*Module)

	if !mod.outputFile.Valid() {

		t.Errorf("%q must have output\n", name)

		return

	}

	snapshotPath := filepath.Join(subDir, mod.outputFile.Path().Base())



	out := vndkSnapshot.Output(snapshotPath)

	if out.Input != mod.outputFile.Path() {

cc/vndk.go

+190 −116
Original line number Diff line number Diff line
@@ -19,6 +19,7 @@ import ( 
	"errors"

	"fmt"

	"path/filepath"

	"sort"

	"strings"

	"sync"
@@ -199,8 +200,6 @@ var ( 
	llndkLibrariesKey                = android.NewOnceKey("llndkLibrarires")

	vndkPrivateLibrariesKey          = android.NewOnceKey("vndkPrivateLibrarires")

	vndkUsingCoreVariantLibrariesKey = android.NewOnceKey("vndkUsingCoreVariantLibrarires")

	modulePathsKey                   = android.NewOnceKey("modulePaths")

	vndkSnapshotOutputsKey           = android.NewOnceKey("vndkSnapshotOutputs")

	vndkMustUseVendorVariantListKey  = android.NewOnceKey("vndkMustUseVendorVariantListKey")

	vndkLibrariesLock                sync.Mutex
@@ -247,18 +246,6 @@ func vndkUsingCoreVariantLibraries(config android.Config) map[string]string { 
	}).(map[string]string)

}



func modulePaths(config android.Config) map[string]string {

	return config.Once(modulePathsKey, func() interface{} {

		return make(map[string]string)

	}).(map[string]string)

}



func vndkSnapshotOutputs(config android.Config) *android.RuleBuilderInstalls {

	return config.Once(vndkSnapshotOutputsKey, func() interface{} {

		return &android.RuleBuilderInstalls{}

	}).(*android.RuleBuilderInstalls)

}



func vndkMustUseVendorVariantList(cfg android.Config) []string {

	return cfg.Once(vndkMustUseVendorVariantListKey, func() interface{} {

		return config.VndkMustUseVendorVariantList
@@ -297,8 +284,6 @@ func processVndkLibrary(mctx android.BottomUpMutatorContext, m *Module) { 
	vndkLibrariesLock.Lock()

	defer vndkLibrariesLock.Unlock()



	modulePaths(mctx.Config())[name] = mctx.ModuleDir()



	if inList(name, vndkMustUseVendorVariantList(mctx.Config())) {

		m.Properties.MustUseVendorVariant = true

	}
@@ -376,10 +361,6 @@ func VndkMutator(mctx android.BottomUpMutatorContext) { 


func init() {

	android.RegisterSingletonType("vndk-snapshot", VndkSnapshotSingleton)

	android.RegisterMakeVarsProvider(pctx, func(ctx android.MakeVarsContext) {

		outputs := vndkSnapshotOutputs(ctx.Config())

		ctx.Strict("SOONG_VNDK_SNAPSHOT_FILES", outputs.String())

	})

}



func VndkSnapshotSingleton() android.Singleton {
@@ -388,12 +369,13 @@ func VndkSnapshotSingleton() android.Singleton { 


type vndkSnapshotSingleton struct {

	installedLlndkLibraries      []string

	llnkdLibrariesFile           android.Path

	llndkLibrariesFile           android.Path

	vndkSpLibrariesFile          android.Path

	vndkCoreLibrariesFile        android.Path

	vndkPrivateLibrariesFile     android.Path

	vndkCoreVariantLibrariesFile android.Path

	vndkLibrariesFile            android.Path

	vndkSnapshotZipFile          android.OptionalPath

}



func (c *vndkSnapshotSingleton) GenerateBuildActions(ctx android.SingletonContext) {
@@ -413,15 +395,42 @@ func (c *vndkSnapshotSingleton) GenerateBuildActions(ctx android.SingletonContex 
		return

	}



	outputs := vndkSnapshotOutputs(ctx.Config())

	var snapshotOutputs android.Paths



	/*

		VNDK snapshot zipped artifacts directory structure:

		{SNAPSHOT_ARCH}/

			arch-{TARGET_ARCH}-{TARGET_ARCH_VARIANT}/

				shared/

					vndk-core/

						(VNDK-core libraries, e.g. libbinder.so)

					vndk-sp/

						(VNDK-SP libraries, e.g. libc++.so)

			arch-{TARGET_2ND_ARCH}-{TARGET_2ND_ARCH_VARIANT}/

				shared/

					vndk-core/

						(VNDK-core libraries, e.g. libbinder.so)

					vndk-sp/

						(VNDK-SP libraries, e.g. libc++.so)

			binder32/

				(This directory is newly introduced in v28 (Android P) to hold

				prebuilts built for 32-bit binder interface.)

				arch-{TARGET_ARCH}-{TARGE_ARCH_VARIANT}/

					...

			configs/

				(various *.txt configuration files)

			include/

				(header files of same directory structure with source tree)

			NOTICE_FILES/

				(notice files of libraries, e.g. libcutils.so.txt)

	*/



	snapshotDir := "vndk-snapshot"

	snapshotArchDir := filepath.Join(snapshotDir, ctx.DeviceConfig().DeviceArch())



	vndkLibDir := make(map[android.ArchType]string)



	snapshotVariantDir := ctx.DeviceConfig().DeviceArch()

	targetArchDirMap := make(map[android.ArchType]string)

	for _, target := range ctx.Config().Targets[android.Android] {

		dir := snapshotVariantDir

		dir := snapshotArchDir

		if ctx.DeviceConfig().BinderBitness() == "32" {

			dir = filepath.Join(dir, "binder32")

		}
@@ -430,64 +439,55 @@ func (c *vndkSnapshotSingleton) GenerateBuildActions(ctx android.SingletonContex 
			arch += "-" + target.Arch.ArchVariant

		}

		dir = filepath.Join(dir, arch)

		vndkLibDir[target.Arch.ArchType] = dir

		targetArchDirMap[target.Arch.ArchType] = dir

	}

	configsDir := filepath.Join(snapshotVariantDir, "configs")

	noticeDir := filepath.Join(snapshotVariantDir, "NOTICE_FILES")

	includeDir := filepath.Join(snapshotVariantDir, "include")

	configsDir := filepath.Join(snapshotArchDir, "configs")

	noticeDir := filepath.Join(snapshotArchDir, "NOTICE_FILES")

	includeDir := filepath.Join(snapshotArchDir, "include")



	// set of include paths exported by VNDK libraries

	exportedIncludes := make(map[string]bool)



	// generated header files among exported headers.

	var generatedHeaders android.Paths



	// set of notice files copied.

	noticeBuilt := make(map[string]bool)



	installSnapshotFileFromPath := func(path android.Path, out string) {

	// paths of VNDK modules for GPL license checking

	modulePaths := make(map[string]string)



	// actual module names of .so files

	// e.g. moduleNames["libprotobuf-cpp-full-3.9.1.so"] = "libprotobuf-cpp-full"

	moduleNames := make(map[string]string)



	installSnapshotFileFromPath := func(path android.Path, out string) android.OutputPath {

		outPath := android.PathForOutput(ctx, out)

		ctx.Build(pctx, android.BuildParams{

			Rule:        android.Cp,

			Input:       path,

			Output:      android.PathForOutput(ctx, snapshotDir, out),

			Output:      outPath,

			Description: "vndk snapshot " + out,

			Args: map[string]string{

				"cpFlags": "-f -L",

			},

		})

		*outputs = append(*outputs, android.RuleBuilderInstall{

			From: android.PathForOutput(ctx, snapshotDir, out),

			To:   out,

		})

		return outPath

	}

	installSnapshotFileFromContent := func(content, out string) {



	installSnapshotFileFromContent := func(content, out string) android.OutputPath {

		outPath := android.PathForOutput(ctx, out)

		ctx.Build(pctx, android.BuildParams{

			Rule:        android.WriteFile,

			Output:      android.PathForOutput(ctx, snapshotDir, out),

			Output:      outPath,

			Description: "vndk snapshot " + out,

			Args: map[string]string{

				"content": content,

			},

		})

		*outputs = append(*outputs, android.RuleBuilderInstall{

			From: android.PathForOutput(ctx, snapshotDir, out),

			To:   out,

		})

	}



	tryBuildNotice := func(m *Module) {

		name := ctx.ModuleName(m) + ".so.txt"



		if _, ok := noticeBuilt[name]; ok {

			return

		}



		noticeBuilt[name] = true



		if m.NoticeFile().Valid() {

			installSnapshotFileFromPath(m.NoticeFile().Path(), filepath.Join(noticeDir, name))

		}

		return outPath

	}



	vndkCoreLibraries := android.SortedStringKeys(vndkCoreLibraries(ctx.Config()))

	vndkSpLibraries := android.SortedStringKeys(vndkSpLibraries(ctx.Config()))

	vndkPrivateLibraries := android.SortedStringKeys(vndkPrivateLibraries(ctx.Config()))



	var generatedHeaders android.Paths

	includeDirs := make(map[string]bool)



	type vndkSnapshotLibraryInterface interface {

		exportedFlagsProducer

		libraryInterface
@@ -496,12 +496,31 @@ func (c *vndkSnapshotSingleton) GenerateBuildActions(ctx android.SingletonContex 
	var _ vndkSnapshotLibraryInterface = (*prebuiltLibraryLinker)(nil)

	var _ vndkSnapshotLibraryInterface = (*libraryDecorator)(nil)



	installVndkSnapshotLib := func(m *Module, l vndkSnapshotLibraryInterface, dir string) bool {

		name := ctx.ModuleName(m)

		libOut := filepath.Join(dir, name+".so")

	installVndkSnapshotLib := func(m *Module, l vndkSnapshotLibraryInterface, vndkType string) (android.Paths, bool) {

		targetArchDir, ok := targetArchDirMap[m.Target().Arch.ArchType]

		if !ok {

			return nil, false

		}



		var ret android.Paths



		libPath := m.outputFile.Path()

		stem := libPath.Base()

		snapshotLibOut := filepath.Join(targetArchDir, "shared", vndkType, stem)

		ret = append(ret, installSnapshotFileFromPath(libPath, snapshotLibOut))



		installSnapshotFileFromPath(m.outputFile.Path(), libOut)

		tryBuildNotice(m)

		moduleNames[stem] = ctx.ModuleName(m)

		modulePaths[stem] = ctx.ModuleDir(m)



		if m.NoticeFile().Valid() {

			noticeName := stem + ".txt"

			// skip already copied notice file

			if _, ok := noticeBuilt[noticeName]; !ok {

				noticeBuilt[noticeName] = true

				ret = append(ret, installSnapshotFileFromPath(

					m.NoticeFile().Path(), filepath.Join(noticeDir, noticeName)))

			}

		}



		if ctx.Config().VndkSnapshotBuildArtifacts() {

			prop := struct {
@@ -515,20 +534,19 @@ func (c *vndkSnapshotSingleton) GenerateBuildActions(ctx android.SingletonContex 
			prop.ExportedSystemDirs = l.exportedSystemDirs().Strings()

			prop.RelativeInstallPath = m.RelativeInstallPath()



			propOut := libOut + ".json"

			propOut := snapshotLibOut + ".json"



			j, err := json.Marshal(prop)

			if err != nil {

				ctx.Errorf("json marshal to %q failed: %#v", propOut, err)

				return false

				return nil, false

			}



			installSnapshotFileFromContent(string(j), propOut)

			ret = append(ret, installSnapshotFileFromContent(string(j), propOut))

		}

		return true

		return ret, true

	}



	isVndkSnapshotLibrary := func(m *Module) (i vndkSnapshotLibraryInterface, libDir string, isVndkSnapshotLib bool) {

	isVndkSnapshotLibrary := func(m *Module) (i vndkSnapshotLibraryInterface, vndkType string, isVndkSnapshotLib bool) {

		if m.Target().NativeBridge == android.NativeBridgeEnabled {

			return nil, "", false

		}
@@ -539,14 +557,15 @@ func (c *vndkSnapshotSingleton) GenerateBuildActions(ctx android.SingletonContex 
		if !ok || !l.shared() {

			return nil, "", false

		}

		name := ctx.ModuleName(m)

		if inList(name, vndkCoreLibraries) {

			return l, filepath.Join("shared", "vndk-core"), true

		} else if inList(name, vndkSpLibraries) {

			return l, filepath.Join("shared", "vndk-sp"), true

		if m.vndkVersion() == ctx.DeviceConfig().PlatformVndkVersion() && m.IsVndk() && !m.isVndkExt() {

			if m.isVndkSp() {

				return l, "vndk-sp", true

			} else {

			return nil, "", false

				return l, "vndk-core", true

			}

		}



		return nil, "", false

	}



	ctx.VisitAllModules(func(module android.Module) {
@@ -555,31 +574,32 @@ func (c *vndkSnapshotSingleton) GenerateBuildActions(ctx android.SingletonContex 
			return

		}



		baseDir, ok := vndkLibDir[m.Target().Arch.ArchType]

		l, vndkType, ok := isVndkSnapshotLibrary(m)

		if !ok {

			return

		}



		l, libDir, ok := isVndkSnapshotLibrary(m)

		libs, ok := installVndkSnapshotLib(m, l, vndkType)

		if !ok {

			return

		}



		if !installVndkSnapshotLib(m, l, filepath.Join(baseDir, libDir)) {

			return

		}

		snapshotOutputs = append(snapshotOutputs, libs...)



		// We glob headers from include directories inside source tree. So we first gather

		// all include directories inside our source tree. On the contrast, we manually

		// collect generated headers from dependencies as they can't globbed.

		generatedHeaders = append(generatedHeaders, l.exportedDeps()...)

		for _, dir := range append(l.exportedDirs(), l.exportedSystemDirs()...) {

			includeDirs[dir.String()] = true

			exportedIncludes[dir.String()] = true

		}

	})



	if ctx.Config().VndkSnapshotBuildArtifacts() {

		headers := make(map[string]bool)

		globbedHeaders := make(map[string]bool)



		for _, dir := range android.SortedStringKeys(includeDirs) {

			// workaround to determine if dir is under output directory

		for _, dir := range android.SortedStringKeys(exportedIncludes) {

			// Skip if dir is for generated headers

			if strings.HasPrefix(dir, android.PathForOutput(ctx).String()) {

				continue

			}
@@ -598,14 +618,14 @@ func (c *vndkSnapshotSingleton) GenerateBuildActions(ctx android.SingletonContex 
					if strings.HasSuffix(header, "/") {

						continue

					}

					headers[header] = true

					globbedHeaders[header] = true

				}

			}

		}



		for _, header := range android.SortedStringKeys(headers) {

			installSnapshotFileFromPath(android.PathForSource(ctx, header),

				filepath.Join(includeDir, header))

		for _, header := range android.SortedStringKeys(globbedHeaders) {

			snapshotOutputs = append(snapshotOutputs, installSnapshotFileFromPath(

				android.PathForSource(ctx, header), filepath.Join(includeDir, header)))

		}



		isHeader := func(path string) bool {
@@ -617,6 +637,7 @@ func (c *vndkSnapshotSingleton) GenerateBuildActions(ctx android.SingletonContex 
			return false

		}



		// For generated headers, manually install one by one, rather than glob

		for _, path := range android.PathsToDirectorySortedPaths(android.FirstUniquePaths(generatedHeaders)) {

			header := path.String()
@@ -624,33 +645,85 @@ func (c *vndkSnapshotSingleton) GenerateBuildActions(ctx android.SingletonContex 
				continue

			}



			installSnapshotFileFromPath(path, filepath.Join(includeDir, header))

			snapshotOutputs = append(snapshotOutputs, installSnapshotFileFromPath(

				path, filepath.Join(includeDir, header)))

		}

	}



	installSnapshotFileFromContent(android.JoinWithSuffix(vndkCoreLibraries, ".so", "\\n"),

		filepath.Join(configsDir, "vndkcore.libraries.txt"))

	installSnapshotFileFromContent(android.JoinWithSuffix(vndkPrivateLibraries, ".so", "\\n"),

		filepath.Join(configsDir, "vndkprivate.libraries.txt"))

	snapshotOutputs = append(snapshotOutputs,

		installSnapshotFileFromPath(c.vndkCoreLibrariesFile, filepath.Join(configsDir, "vndkcore.libraries.txt")),

		installSnapshotFileFromPath(c.vndkPrivateLibrariesFile, filepath.Join(configsDir, "vndkprivate.libraries.txt")),

		installSnapshotFileFromPath(c.vndkSpLibrariesFile, filepath.Join(configsDir, "vndksp.libraries.txt")),

		installSnapshotFileFromPath(c.llndkLibrariesFile, filepath.Join(configsDir, "llndk.libraries.txt")),

	)



	var modulePathTxtBuilder strings.Builder

	/*

		Dump a map to a list file as:



		{key1} {value1}

		{key2} {value2}

		...

	*/

	installMapListFile := func(m map[string]string, path string) android.OutputPath {

		var txtBuilder strings.Builder

		for idx, k := range android.SortedStringKeys(m) {

			if idx > 0 {

				txtBuilder.WriteString("\\n")

			}

			txtBuilder.WriteString(k)

			txtBuilder.WriteString(" ")

			txtBuilder.WriteString(m[k])

		}

		return installSnapshotFileFromContent(txtBuilder.String(), path)

	}



	/*

		module_paths.txt contains paths on which VNDK modules are defined.

		e.g.,

			libbase.so system/core/base

			libc.so bionic/libc

			...

	*/

	snapshotOutputs = append(snapshotOutputs, installMapListFile(modulePaths, filepath.Join(configsDir, "module_paths.txt")))



	/*

		module_names.txt contains names as which VNDK modules are defined,

		because output filename and module name can be different with stem and suffix properties.



		e.g.,

			libcutils.so libcutils

			libprotobuf-cpp-full-3.9.2.so libprotobuf-cpp-full

			...

	*/

	snapshotOutputs = append(snapshotOutputs, installMapListFile(moduleNames, filepath.Join(configsDir, "module_names.txt")))



	// All artifacts are ready. Sort them to normalize ninja and then zip.

	sort.Slice(snapshotOutputs, func(i, j int) bool {

		return snapshotOutputs[i].String() < snapshotOutputs[j].String()

	})



	modulePaths := modulePaths(ctx.Config())

	zipPath := android.PathForOutput(ctx, snapshotDir, "android-vndk-"+ctx.DeviceConfig().DeviceArch()+".zip")

	zipRule := android.NewRuleBuilder()



	first := true

	for _, lib := range android.SortedStringKeys(modulePaths) {

		if first {

			first = false

		} else {

			modulePathTxtBuilder.WriteString("\\n")

		}

		modulePathTxtBuilder.WriteString(lib)

		modulePathTxtBuilder.WriteString(".so ")

		modulePathTxtBuilder.WriteString(modulePaths[lib])

	}

	// If output files are too many, soong_zip command can exceed ARG_MAX.

	// So first dump file lists into a single list file, and then feed it to Soong

	snapshotOutputList := android.PathForOutput(ctx, snapshotDir, "android-vndk-"+ctx.DeviceConfig().DeviceArch()+"_list")

	zipRule.Command().

		Text("( xargs").

		FlagWithRspFileInputList("-n1 echo < ", snapshotOutputs).

		FlagWithOutput("| tr -d \\' > ", snapshotOutputList).

		Text(")")



	zipRule.Temporary(snapshotOutputList)



	zipRule.Command().

		BuiltTool(ctx, "soong_zip").

		FlagWithOutput("-o ", zipPath).

		FlagWithArg("-C ", android.PathForOutput(ctx, snapshotDir).String()).

		FlagWithInput("-l ", snapshotOutputList)



	installSnapshotFileFromContent(modulePathTxtBuilder.String(),

		filepath.Join(configsDir, "module_paths.txt"))

	zipRule.Build(pctx, ctx, zipPath.String(), "vndk snapshot "+zipPath.String())

	c.vndkSnapshotZipFile = android.OptionalPathForPath(zipPath)

}



func getVndkFileName(m *Module) (string, error) {
@@ -697,7 +770,7 @@ func (c *vndkSnapshotSingleton) buildVndkLibrariesTxtFiles(ctx android.Singleton 
	vndkprivate := android.SortedStringMapValues(vndkPrivateLibraries(ctx.Config()))

	vndkcorevariant := android.SortedStringMapValues(vndkUsingCoreVariantLibraries(ctx.Config()))



	c.llnkdLibrariesFile = installListFile(llndk, "llndk.libraries.txt")

	c.llndkLibrariesFile = installListFile(llndk, "llndk.libraries.txt")

	c.vndkCoreLibrariesFile = installListFile(vndkcore, "vndkcore.libraries.txt")

	c.vndkSpLibrariesFile = installListFile(vndksp, "vndksp.libraries.txt")

	c.vndkPrivateLibrariesFile = installListFile(vndkprivate, "vndkprivate.libraries.txt")
@@ -739,11 +812,12 @@ func (c *vndkSnapshotSingleton) MakeVars(ctx android.MakeVarsContext) { 
	ctx.Strict("VNDK_PRIVATE_LIBRARIES", strings.Join(android.SortedStringKeys(vndkPrivateLibraries(ctx.Config())), " "))

	ctx.Strict("VNDK_USING_CORE_VARIANT_LIBRARIES", strings.Join(android.SortedStringKeys(vndkUsingCoreVariantLibraries(ctx.Config())), " "))



	ctx.Strict("LLNDK_LIBRARIES_FILE", c.llnkdLibrariesFile.String())

	ctx.Strict("LLNDK_LIBRARIES_FILE", c.llndkLibrariesFile.String())

	ctx.Strict("VNDKCORE_LIBRARIES_FILE", c.vndkCoreLibrariesFile.String())

	ctx.Strict("VNDKSP_LIBRARIES_FILE", c.vndkSpLibrariesFile.String())

	ctx.Strict("VNDKPRIVATE_LIBRARIES_FILE", c.vndkPrivateLibrariesFile.String())

	ctx.Strict("VNDKCOREVARIANT_LIBRARIES_FILE", c.vndkCoreVariantLibrariesFile.String())



	ctx.Strict("VNDK_LIBRARIES_FILE", c.vndkLibrariesFile.String())

	ctx.Strict("SOONG_VNDK_SNAPSHOT_ZIP", c.vndkSnapshotZipFile.String())

}