Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit ac775b2a authored by Inseob Kim's avatar Inseob Kim
Browse files

Capture snapshot headers in parallel 

VNDK and vendor snapshot singleton work in a single thread, so globbing
in singleton results in ridiculus running time. Moving codes to
GenerateAndroidBuildActions to reduce running time.

Bug: 150406226
Test: VNDK_SNAPSHOT_BUILD_ARTIFACTS=true m dist vndk vendor-snapshot
Test: vendorSnapshotSingleton build time became 0.56s (from 10s)
Test: build.ninja building time became 1m11s (from 1m21s)
Change-Id: I4a081eef5847c62ca00280ca426f5b4e10f87b59
Merged-In: I4a081eef5847c62ca00280ca426f5b4e10f87b59
(cherry picked from commit eda2e9c7)
parent 307dd9f4

cc/cc.go

+7 −0
Original line number Diff line number Diff line
@@ -1465,6 +1465,13 @@ func (c *Module) GenerateAndroidBuildActions(actx android.ModuleContext) { 
			c.Properties.HideFromMake = false // unhide

			// Note: this is still non-installable

		}



		// glob exported headers for snapshot, if BOARD_VNDK_VERSION is current.

		if i, ok := c.linker.(snapshotLibraryInterface); ok && ctx.DeviceConfig().VndkVersion() == "current" {

			if isSnapshotAware(ctx, c) {

				i.collectHeadersForSnapshot(ctx)

			}

		}

	}



	if c.installable() {

cc/library.go

+62 −0
Original line number Diff line number Diff line
@@ -390,6 +390,68 @@ type libraryDecorator struct { 
	*baseCompiler

	*baseLinker

	*baseInstaller



	collectedSnapshotHeaders android.Paths

}



// collectHeadersForSnapshot collects all exported headers from library.

// It globs header files in the source tree for exported include directories,

// and tracks generated header files separately.

//

// This is to be called from GenerateAndroidBuildActions, and then collected

// header files can be retrieved by snapshotHeaders().

func (l *libraryDecorator) collectHeadersForSnapshot(ctx android.ModuleContext) {

	ret := android.Paths{}



	// Headers in the source tree should be globbed. On the contrast, generated headers

	// can't be globbed, and they should be manually collected.

	// So, we first filter out intermediate directories (which contains generated headers)

	// from exported directories, and then glob headers under remaining directories.

	for _, path := range append(l.exportedDirs(), l.exportedSystemDirs()...) {

		dir := path.String()

		// Skip if dir is for generated headers

		if strings.HasPrefix(dir, android.PathForOutput(ctx).String()) {

			continue

		}

		exts := headerExts

		// Glob all files under this special directory, because of C++ headers.

		if strings.HasPrefix(dir, "external/libcxx/include") {

			exts = []string{""}

		}

		for _, ext := range exts {

			glob, err := ctx.GlobWithDeps(dir+"/**/*"+ext, nil)

			if err != nil {

				ctx.ModuleErrorf("glob failed: %#v", err)

				return

			}

			for _, header := range glob {

				if strings.HasSuffix(header, "/") {

					continue

				}

				ret = append(ret, android.PathForSource(ctx, header))

			}

		}

	}



	// Collect generated headers

	for _, header := range append(l.exportedGeneratedHeaders(), l.exportedDeps()...) {

		// TODO(b/148123511): remove exportedDeps after cleaning up genrule

		if strings.HasSuffix(header.Base(), "-phony") {

			continue

		}

		ret = append(ret, header)

	}



	l.collectedSnapshotHeaders = ret

}



// This returns all exported header files, both generated ones and headers from source tree.

// collectHeadersForSnapshot() must be called before calling this.

func (l *libraryDecorator) snapshotHeaders() android.Paths {

	if l.collectedSnapshotHeaders == nil {

		panic("snapshotHeaders() must be called after collectHeadersForSnapshot()")

	}

	return l.collectedSnapshotHeaders

}



func (library *libraryDecorator) linkerProps() []interface{} {

cc/snapshot_utils.go

+8 −44
Original line number Diff line number Diff line
@@ -14,8 +14,6 @@ 
package cc



import (

	"strings"



	"android/soong/android"

)
@@ -26,6 +24,8 @@ var ( 
type snapshotLibraryInterface interface {

	exportedFlagsProducer

	libraryInterface

	collectHeadersForSnapshot(ctx android.ModuleContext)

	snapshotHeaders() android.Paths

}



var _ snapshotLibraryInterface = (*prebuiltLibraryLinker)(nil)
@@ -58,49 +58,13 @@ func (s *snapshotMap) get(name string, arch android.ArchType) (snapshot string, 
	return snapshot, found

}



func exportedHeaders(ctx android.SingletonContext, l exportedFlagsProducer) android.Paths {

	var ret android.Paths



	// Headers in the source tree should be globbed. On the contrast, generated headers

	// can't be globbed, and they should be manually collected.

	// So, we first filter out intermediate directories (which contains generated headers)

	// from exported directories, and then glob headers under remaining directories.

	for _, path := range append(l.exportedDirs(), l.exportedSystemDirs()...) {

		dir := path.String()

		// Skip if dir is for generated headers

		if strings.HasPrefix(dir, android.PathForOutput(ctx).String()) {

			continue

		}

		exts := headerExts

		// Glob all files under this special directory, because of C++ headers.

		if strings.HasPrefix(dir, "external/libcxx/include") {

			exts = []string{""}

		}

		for _, ext := range exts {

			glob, err := ctx.GlobWithDeps(dir+"/**/*"+ext, nil)

			if err != nil {

				ctx.Errorf("%#v\n", err)

				return nil

			}

			for _, header := range glob {

				if strings.HasSuffix(header, "/") {

					continue

				}

				ret = append(ret, android.PathForSource(ctx, header))

			}

		}

	}



	// Collect generated headers

	for _, header := range append(l.exportedGeneratedHeaders(), l.exportedDeps()...) {

		// TODO(b/148123511): remove exportedDeps after cleaning up genrule

		if strings.HasSuffix(header.Base(), "-phony") {

			continue

		}

		ret = append(ret, header)

func isSnapshotAware(ctx android.ModuleContext, m *Module) bool {

	if _, _, ok := isVndkSnapshotLibrary(ctx.DeviceConfig(), m); ok {

		return ctx.Config().VndkSnapshotBuildArtifacts()

	} else if isVendorSnapshotModule(m, ctx.ModuleDir()) {

		return true

	}



	return ret

	return false

}



func copyFile(ctx android.SingletonContext, path android.Path, out string) android.OutputPath {

cc/vendor_snapshot.go

+11 −14
Original line number Diff line number Diff line
@@ -428,12 +428,12 @@ func isVendorProprietaryPath(dir string) bool { 
// AOSP. They are not guaranteed to be compatible with older vendor images. (e.g. might

// depend on newer VNDK) So they are captured as vendor snapshot To build older vendor

// image and newer system image altogether.

func isVendorSnapshotModule(ctx android.SingletonContext, m *Module) bool {

func isVendorSnapshotModule(m *Module, moduleDir string) bool {

	if !m.Enabled() {

		return false

	}

	// skip proprietary modules, but include all VNDK (static)

	if isVendorProprietaryPath(ctx.ModuleDir(m)) && !m.IsVndk() {

	if isVendorProprietaryPath(moduleDir) && !m.IsVndk() {

		return false

	}

	if m.Target().Os.Class != android.Device {
@@ -525,14 +525,6 @@ func (c *vendorSnapshotSingleton) GenerateBuildActions(ctx android.SingletonCont 


	var headers android.Paths



	type vendorSnapshotLibraryInterface interface {

		exportedFlagsProducer

		libraryInterface

	}



	var _ vendorSnapshotLibraryInterface = (*prebuiltLibraryLinker)(nil)

	var _ vendorSnapshotLibraryInterface = (*libraryDecorator)(nil)



	installSnapshot := func(m *Module) android.Paths {

		targetArch := "arch-" + m.Target().Arch.ArchType.String()

		if m.Target().Arch.ArchVariant != "" {
@@ -588,7 +580,7 @@ func (c *vendorSnapshotSingleton) GenerateBuildActions(ctx android.SingletonCont 


		var propOut string



		if l, ok := m.linker.(vendorSnapshotLibraryInterface); ok {

		if l, ok := m.linker.(snapshotLibraryInterface); ok {

			// library flags

			prop.ExportedFlags = l.exportedFlags()

			for _, dir := range l.exportedDirs() {
@@ -652,13 +644,18 @@ func (c *vendorSnapshotSingleton) GenerateBuildActions(ctx android.SingletonCont 


	ctx.VisitAllModules(func(module android.Module) {

		m, ok := module.(*Module)

		if !ok || !isVendorSnapshotModule(ctx, m) {

		if !ok {

			return

		}



		moduleDir := ctx.ModuleDir(module)

		if !isVendorSnapshotModule(m, moduleDir) {

			return

		}



		snapshotOutputs = append(snapshotOutputs, installSnapshot(m)...)

		if l, ok := m.linker.(vendorSnapshotLibraryInterface); ok {

			headers = append(headers, exportedHeaders(ctx, l)...)

		if l, ok := m.linker.(snapshotLibraryInterface); ok {

			headers = append(headers, l.snapshotHeaders()...)

		}



		if m.NoticeFile().Valid() {

cc/vndk.go

+24 −24
Original line number Diff line number Diff line
@@ -496,6 +496,28 @@ type vndkSnapshotSingleton struct { 
	vndkSnapshotZipFile android.OptionalPath

}



func isVndkSnapshotLibrary(config android.DeviceConfig, m *Module) (i snapshotLibraryInterface, vndkType string, isVndkSnapshotLib bool) {

	if m.Target().NativeBridge == android.NativeBridgeEnabled {

		return nil, "", false

	}

	if !m.inVendor() || !m.installable() || m.isSnapshotPrebuilt() {

		return nil, "", false

	}

	l, ok := m.linker.(snapshotLibraryInterface)

	if !ok || !l.shared() {

		return nil, "", false

	}

	if m.VndkVersion() == config.PlatformVndkVersion() && m.IsVndk() && !m.isVndkExt() {

		if m.isVndkSp() {

			return l, "vndk-sp", true

		} else {

			return l, "vndk-core", true

		}

	}



	return nil, "", false

}



func (c *vndkSnapshotSingleton) GenerateBuildActions(ctx android.SingletonContext) {

	// build these files even if PlatformVndkVersion or BoardVndkVersion is not set

	c.buildVndkLibrariesTxtFiles(ctx)
@@ -598,35 +620,13 @@ func (c *vndkSnapshotSingleton) GenerateBuildActions(ctx android.SingletonContex 
		return ret, true

	}



	isVndkSnapshotLibrary := func(m *Module) (i snapshotLibraryInterface, vndkType string, isVndkSnapshotLib bool) {

		if m.Target().NativeBridge == android.NativeBridgeEnabled {

			return nil, "", false

		}

		if !m.inVendor() || !m.installable() || m.isSnapshotPrebuilt() {

			return nil, "", false

		}

		l, ok := m.linker.(snapshotLibraryInterface)

		if !ok || !l.shared() {

			return nil, "", false

		}

		if m.VndkVersion() == ctx.DeviceConfig().PlatformVndkVersion() && m.IsVndk() && !m.isVndkExt() {

			if m.isVndkSp() {

				return l, "vndk-sp", true

			} else {

				return l, "vndk-core", true

			}

		}



		return nil, "", false

	}



	ctx.VisitAllModules(func(module android.Module) {

		m, ok := module.(*Module)

		if !ok || !m.Enabled() {

			return

		}



		l, vndkType, ok := isVndkSnapshotLibrary(m)

		l, vndkType, ok := isVndkSnapshotLibrary(ctx.DeviceConfig(), m)

		if !ok {

			return

		}
@@ -655,7 +655,7 @@ func (c *vndkSnapshotSingleton) GenerateBuildActions(ctx android.SingletonContex 
		}



		if ctx.Config().VndkSnapshotBuildArtifacts() {

			headers = append(headers, exportedHeaders(ctx, l)...)

			headers = append(headers, l.snapshotHeaders()...)

		}

	})