Merge "Create soong-generated filesystem diff test" into main (e390c0e4) · Commits · e / os / android_build_soong

filesystem/filesystem.go

+23 −2

Original line number	Diff line number	Diff line
		@@ -35,7 +35,7 @@ func init() {
		}

		func registerBuildComponents(ctx android.RegistrationContext) {
		ctx.RegisterModuleType("android_filesystem", filesystemFactory)
		ctx.RegisterModuleType("android_filesystem", FilesystemFactory)
		ctx.RegisterModuleType("android_filesystem_defaults", filesystemDefaultsFactory)
		ctx.RegisterModuleType("android_system_image", SystemImageFactory)
		ctx.RegisterModuleType("avb_add_hash_footer", avbAddHashFooterFactory)
		@@ -137,6 +137,12 @@ type FilesystemProperties struct {
		Gen_aconfig_flags_pb *bool

		Fsverity fsverityProperties

		// If this property is set to true, the filesystem will call ctx.UncheckedModule(), causing
		// it to not be built on checkbuilds. Used for the automatic migration from make to soong
		// build modules, where we want to emit some not-yet-working filesystems and we don't want them
		// to be built.
		Unchecked_module *bool `blueprint:"mutated"`
		}

		// android_filesystem packages a set of modules and their transitive dependencies into a filesystem
		@@ -144,7 +150,7 @@ type FilesystemProperties struct {
		// modules in the filesystem image are built for the target device (i.e. Android, not Linux host).
		// The modules are placed in the filesystem image just like they are installed to the ordinary
		// partitions like system.img. For example, cc_library modules are placed under ./lib[64] directory.
		func filesystemFactory() android.Module {
		func FilesystemFactory() android.Module {
		module := &filesystem{}
		module.filterPackagingSpec = module.filterInstallablePackagingSpec
		initFilesystemModule(module, module)
		@@ -177,6 +183,13 @@ const (
		unknown
		)

		type FilesystemInfo struct {
		// A text file containing the list of paths installed on the partition.
		FileListFile android.Path
		}

		var FilesystemProvider = blueprint.NewProvider[FilesystemInfo]()

		func (f *filesystem) fsType(ctx android.ModuleContext) fsType {
		typeStr := proptools.StringDefault(f.properties.Type, "ext4")
		switch typeStr {
		@@ -227,6 +240,14 @@ func (f *filesystem) GenerateAndroidBuildActions(ctx android.ModuleContext) {

		f.fileListFile = android.PathForModuleOut(ctx, "fileList").OutputPath
		android.WriteFileRule(ctx, f.fileListFile, f.installedFilesList())

		android.SetProvider(ctx, FilesystemProvider, FilesystemInfo{
		FileListFile: f.fileListFile,
		})

		if proptools.Bool(f.properties.Unchecked_module) {
		ctx.UncheckedModule()
		}
		}

		func (f *filesystem) appendToEntry(ctx android.ModuleContext, installedFile android.OutputPath) {

fsgen/Android.bp

+4 −0

Original line number	Diff line number	Diff line
		@@ -19,3 +19,7 @@ bootstrap_go_package {
		],
		pluginFor: ["soong_build"],
		}

		soong_filesystem_creator {
		name: "soong_filesystem_creator",
		}

fsgen/filesystem_creator.go

+118 −12

Original line number	Diff line number	Diff line
		@@ -17,12 +17,16 @@ package fsgen
		import (
		"android/soong/android"
		"android/soong/filesystem"
		"crypto/sha256"
		"fmt"
		"strconv"

		"github.com/google/blueprint"
		"github.com/google/blueprint/proptools"
		)

		var pctx = android.NewPackageContext("android/soong/fsgen")

		func init() {
		registerBuildComponents(android.InitRegistrationContext)
		}
		@@ -31,14 +35,22 @@ func registerBuildComponents(ctx android.RegistrationContext) {
		ctx.RegisterModuleType("soong_filesystem_creator", filesystemCreatorFactory)
		}

		type filesystemCreatorProps struct {
		Generated_partition_types []string `blueprint:"mutated"`
		Unsupported_partition_types []string `blueprint:"mutated"`
		}

		type filesystemCreator struct {
		android.ModuleBase

		properties filesystemCreatorProps
		}

		func filesystemCreatorFactory() android.Module {
		module := &filesystemCreator{}

		android.InitAndroidModule(module)
		module.AddProperties(&module.properties)
		android.AddLoadHook(module, func(ctx android.LoadHookContext) {
		module.createInternalModules(ctx)
		})
		@@ -47,36 +59,62 @@ func filesystemCreatorFactory() android.Module {
		}

		func (f *filesystemCreator) createInternalModules(ctx android.LoadHookContext) {
		f.createSystemImage(ctx)
		for _, partitionType := range []string{"system"} {
		if f.createPartition(ctx, partitionType) {
		f.properties.Generated_partition_types = append(f.properties.Generated_partition_types, partitionType)
		} else {
		f.properties.Unsupported_partition_types = append(f.properties.Unsupported_partition_types, partitionType)
		}
		}
		}

		func (f *filesystemCreator) generatedModuleNameForPartition(cfg android.Config, partitionType string) string {
		prefix := "soong"
		if cfg.HasDeviceProduct() {
		prefix = cfg.DeviceProduct()
		}
		return fmt.Sprintf("%s_generated_%s_image", prefix, partitionType)
		}

		func (f *filesystemCreator) createSystemImage(ctx android.LoadHookContext) {
		// Creates a soong module to build the given partition. Returns false if we can't support building
		// it.
		func (f *filesystemCreator) createPartition(ctx android.LoadHookContext, partitionType string) bool {
		baseProps := &struct {
		Name *string
		}{
		Name: proptools.StringPtr(fmt.Sprintf("%s_generated_system_image", ctx.Config().DeviceProduct())),
		Name: proptools.StringPtr(f.generatedModuleNameForPartition(ctx.Config(), partitionType)),
		}

		fsProps := &(filesystem.FilesystemProperties{})
		fsProps := &filesystem.FilesystemProperties{}

		// Don't build this module on checkbuilds, the soong-built partitions are still in-progress
		// and sometimes don't build.
		fsProps.Unchecked_module = proptools.BoolPtr(true)

		partitionVars := ctx.Config().ProductVariables().PartitionVarsForSoongMigrationOnlyDoNotUse
		systemPartitionVars := partitionVars.PartitionQualifiedVariables["system"]
		specificPartitionVars := partitionVars.PartitionQualifiedVariables[partitionType]

		// BOARD_AVB_ENABLE
		fsProps.Use_avb = proptools.BoolPtr(partitionVars.BoardAvbEnable)
		// BOARD_AVB_KEY_PATH
		fsProps.Avb_private_key = proptools.StringPtr(systemPartitionVars.BoardAvbKeyPath)
		fsProps.Avb_private_key = proptools.StringPtr(specificPartitionVars.BoardAvbKeyPath)
		// BOARD_AVB_ALGORITHM
		fsProps.Avb_algorithm = proptools.StringPtr(systemPartitionVars.BoardAvbAlgorithm)
		fsProps.Avb_algorithm = proptools.StringPtr(specificPartitionVars.BoardAvbAlgorithm)
		// BOARD_AVB_SYSTEM_ROLLBACK_INDEX
		if rollbackIndex, err := strconv.ParseInt(systemPartitionVars.BoardAvbRollbackIndex, 10, 64); err == nil {
		if rollbackIndex, err := strconv.ParseInt(specificPartitionVars.BoardAvbRollbackIndex, 10, 64); err == nil {
		fsProps.Rollback_index = proptools.Int64Ptr(rollbackIndex)
		}

		fsProps.Partition_name = proptools.StringPtr("system")
		fsProps.Partition_name = proptools.StringPtr(partitionType)
		// BOARD_SYSTEMIMAGE_FILE_SYSTEM_TYPE
		fsProps.Type = proptools.StringPtr(systemPartitionVars.BoardFileSystemType)
		fsProps.Type = proptools.StringPtr(specificPartitionVars.BoardFileSystemType)
		if *fsProps.Type != "ext4" {
		// Currently the android_filesystem module type only supports ext4:
		// https://cs.android.com/android/platform/superproject/main/+/main:build/soong/filesystem/filesystem.go;l=416;drc=98047cfd07944b297a12d173453bc984806760d2
		return false
		}

		fsProps.Base_dir = proptools.StringPtr("system")
		fsProps.Base_dir = proptools.StringPtr(partitionType)

		fsProps.Gen_aconfig_flags_pb = proptools.BoolPtr(true)

		@@ -103,9 +141,77 @@ func (f *filesystemCreator) createSystemImage(ctx android.LoadHookContext) {
		// - filesystemProperties.Build_logtags
		// - filesystemProperties.Fsverity.Libs
		// - systemImageProperties.Linker_config_src
		ctx.CreateModule(filesystem.SystemImageFactory, baseProps, fsProps)
		var module android.Module
		if partitionType == "system" {
		module = ctx.CreateModule(filesystem.SystemImageFactory, baseProps, fsProps)
		} else {
		module = ctx.CreateModule(filesystem.FilesystemFactory, baseProps, fsProps)
		}
		module.HideFromMake()
		return true
		}

		func (f *filesystemCreator) createDiffTest(ctx android.ModuleContext, partitionType string) android.Path {
		partitionModuleName := f.generatedModuleNameForPartition(ctx.Config(), partitionType)
		systemImage := ctx.GetDirectDepWithTag(partitionModuleName, generatedFilesystemDepTag)
		filesystemInfo, ok := android.OtherModuleProvider(ctx, systemImage, filesystem.FilesystemProvider)
		if !ok {
		ctx.ModuleErrorf("Expected module %s to provide FileysystemInfo", partitionModuleName)
		}
		makeFileList := android.PathForArbitraryOutput(ctx, fmt.Sprintf("target/product/%s/obj/PACKAGING/%s_intermediates/file_list.txt", ctx.Config().DeviceName(), partitionType))
		// For now, don't allowlist anything. The test will fail, but that's fine in the current
		// early stages where we're just figuring out what we need
		emptyAllowlistFile := android.PathForModuleOut(ctx, "allowlist_%s.txt", partitionModuleName)
		android.WriteFileRule(ctx, emptyAllowlistFile, "")
		diffTestResultFile := android.PathForModuleOut(ctx, "diff_test_%s.txt", partitionModuleName)

		builder := android.NewRuleBuilder(pctx, ctx)
		builder.Command().BuiltTool("file_list_diff").
		Input(makeFileList).
		Input(filesystemInfo.FileListFile).
		Input(emptyAllowlistFile).
		Text(partitionModuleName)
		builder.Command().Text("touch").Output(diffTestResultFile)
		builder.Build(partitionModuleName+" diff test", partitionModuleName+" diff test")
		return diffTestResultFile
		}

		func createFailingCommand(ctx android.ModuleContext, message string) android.Path {
		hasher := sha256.New()
		hasher.Write([]byte(message))
		filename := fmt.Sprintf("failing_command_%x.txt", hasher.Sum(nil))
		file := android.PathForModuleOut(ctx, filename)
		builder := android.NewRuleBuilder(pctx, ctx)
		builder.Command().Textf("echo %s", proptools.NinjaAndShellEscape(message))
		builder.Command().Text("exit 1 #").Output(file)
		builder.Build("failing command "+filename, "failing command "+filename)
		return file
		}

		type systemImageDepTagType struct {
		blueprint.BaseDependencyTag
		}

		var generatedFilesystemDepTag systemImageDepTagType

		func (f *filesystemCreator) DepsMutator(ctx android.BottomUpMutatorContext) {
		for _, partitionType := range f.properties.Generated_partition_types {
		ctx.AddDependency(ctx.Module(), generatedFilesystemDepTag, f.generatedModuleNameForPartition(ctx.Config(), partitionType))
		}
		}

		func (f *filesystemCreator) GenerateAndroidBuildActions(ctx android.ModuleContext) {
		if ctx.ModuleDir() != "build/soong/fsgen" {
		ctx.ModuleErrorf("There can only be one soong_filesystem_creator in build/soong/fsgen")
		}
		f.HideFromMake()

		var diffTestFiles []android.Path
		for _, partitionType := range f.properties.Generated_partition_types {
		diffTestFiles = append(diffTestFiles, f.createDiffTest(ctx, partitionType))
		}
		for _, partitionType := range f.properties.Unsupported_partition_types {
		diffTestFiles = append(diffTestFiles, createFailingCommand(ctx, fmt.Sprintf("Couldn't build %s partition", partitionType)))
		}
		ctx.Phony("soong_generated_filesystem_tests", diffTestFiles...)
		}

fsgen/filesystem_creator_test.go

+6 −5

Original line number	Diff line number	Diff line
		@@ -44,12 +44,13 @@ func TestFileSystemCreatorSystemImageProps(t *testing.T) {
		}),
		android.FixtureMergeMockFs(android.MockFS{
		"external/avb/test/data/testkey_rsa4096.pem": nil,
		}),
		).RunTestWithBp(t, `
		"build/soong/fsgen/Android.bp": []byte(`
		soong_filesystem_creator {
		name: "foo",
		}
		`)
		`),
		}),
		).RunTest(t)

		fooSystem := result.ModuleForTests("test_product_generated_system_image", "android_common").Module().(interface {
		FsProps() filesystem.FilesystemProperties