Merge "Refactoring fuzz packaging code" am: 0b45f0b90f (c62f2deb) · Commits · e / os / android_build_soong

cc/Android.bp

+1 −0

Original line number	Diff line number	Diff line
		@@ -58,6 +58,7 @@ bootstrap_go_package {
		"binary.go",
		"binary_sdk_member.go",
		"fuzz.go",
		"fuzz_common.go",
		"library.go",
		"library_headers.go",
		"library_sdk_member.go",

cc/androidmk.go

+8 −8

Original line number	Diff line number	Diff line
		@@ -401,24 +401,24 @@ func (fuzz fuzzBinary) AndroidMkEntries(ctx AndroidMkContext, entries android.
		ctx.subAndroidMk(entries, fuzz.binaryDecorator)

		var fuzzFiles []string
		for _, d := range fuzz.corpus {
		for _, d := range fuzz.fuzzPackagedModule.Corpus {
		fuzzFiles = append(fuzzFiles,
		filepath.Dir(fuzz.corpusIntermediateDir.String())+":corpus/"+d.Base())
		filepath.Dir(fuzz.fuzzPackagedModule.CorpusIntermediateDir.String())+":corpus/"+d.Base())
		}

		for _, d := range fuzz.data {
		for _, d := range fuzz.fuzzPackagedModule.Data {
		fuzzFiles = append(fuzzFiles,
		filepath.Dir(fuzz.dataIntermediateDir.String())+":data/"+d.Rel())
		filepath.Dir(fuzz.fuzzPackagedModule.DataIntermediateDir.String())+":data/"+d.Rel())
		}

		if fuzz.dictionary != nil {
		if fuzz.fuzzPackagedModule.Dictionary != nil {
		fuzzFiles = append(fuzzFiles,
		filepath.Dir(fuzz.dictionary.String())+":"+fuzz.dictionary.Base())
		filepath.Dir(fuzz.fuzzPackagedModule.Dictionary.String())+":"+fuzz.fuzzPackagedModule.Dictionary.Base())
		}

		if fuzz.config != nil {
		if fuzz.fuzzPackagedModule.Config != nil {
		fuzzFiles = append(fuzzFiles,
		filepath.Dir(fuzz.config.String())+":config.json")
		filepath.Dir(fuzz.fuzzPackagedModule.Config.String())+":config.json")
		}

		entries.ExtraEntries = append(entries.ExtraEntries, func(ctx android.AndroidMkExtraEntriesContext, entries *android.AndroidMkEntries) {

cc/cc.go

+3 −4

Original line number	Diff line number	Diff line
		@@ -756,14 +756,13 @@ func IsTestPerSrcDepTag(depTag blueprint.DependencyTag) bool {
		// members of the cc.Module to this decorator. Thus, a cc_binary module has custom linker and
		// installer logic.
		type Module struct {
		android.ModuleBase
		android.DefaultableModuleBase
		android.ApexModuleBase
		FuzzModule

		android.SdkBase
		android.BazelModuleBase

		Properties BaseProperties
		VendorProperties VendorProperties
		Properties BaseProperties

		// initialize before calling Init
		hod android.HostOrDeviceSupported

cc/fuzz.go

+41 −155

Original line number	Diff line number	Diff line
		@@ -94,19 +94,14 @@ type fuzzBinary struct {
		*binaryDecorator
		*baseCompiler

		Properties FuzzProperties
		dictionary android.Path
		corpus android.Paths
		corpusIntermediateDir android.Path
		config android.Path
		data android.Paths
		dataIntermediateDir android.Path
		fuzzPackagedModule FuzzPackagedModule

		installedSharedDeps []string
		}

		func (fuzz *fuzzBinary) linkerProps() []interface{} {
		props := fuzz.binaryDecorator.linkerProps()
		props = append(props, &fuzz.Properties)
		props = append(props, &fuzz.fuzzPackagedModule.FuzzProperties)
		return props
		}

		@@ -257,41 +252,41 @@ func (fuzz *fuzzBinary) install(ctx ModuleContext, file android.Path) {
		"fuzz", ctx.Target().Arch.ArchType.String(), ctx.ModuleName())
		fuzz.binaryDecorator.baseInstaller.install(ctx, file)

		fuzz.corpus = android.PathsForModuleSrc(ctx, fuzz.Properties.Corpus)
		fuzz.fuzzPackagedModule.Corpus = android.PathsForModuleSrc(ctx, fuzz.fuzzPackagedModule.FuzzProperties.Corpus)
		builder := android.NewRuleBuilder(pctx, ctx)
		intermediateDir := android.PathForModuleOut(ctx, "corpus")
		for _, entry := range fuzz.corpus {
		for _, entry := range fuzz.fuzzPackagedModule.Corpus {
		builder.Command().Text("cp").
		Input(entry).
		Output(intermediateDir.Join(ctx, entry.Base()))
		}
		builder.Build("copy_corpus", "copy corpus")
		fuzz.corpusIntermediateDir = intermediateDir
		fuzz.fuzzPackagedModule.CorpusIntermediateDir = intermediateDir

		fuzz.data = android.PathsForModuleSrc(ctx, fuzz.Properties.Data)
		fuzz.fuzzPackagedModule.Data = android.PathsForModuleSrc(ctx, fuzz.fuzzPackagedModule.FuzzProperties.Data)
		builder = android.NewRuleBuilder(pctx, ctx)
		intermediateDir = android.PathForModuleOut(ctx, "data")
		for _, entry := range fuzz.data {
		for _, entry := range fuzz.fuzzPackagedModule.Data {
		builder.Command().Text("cp").
		Input(entry).
		Output(intermediateDir.Join(ctx, entry.Rel()))
		}
		builder.Build("copy_data", "copy data")
		fuzz.dataIntermediateDir = intermediateDir
		fuzz.fuzzPackagedModule.DataIntermediateDir = intermediateDir

		if fuzz.Properties.Dictionary != nil {
		fuzz.dictionary = android.PathForModuleSrc(ctx, *fuzz.Properties.Dictionary)
		if fuzz.dictionary.Ext() != ".dict" {
		if fuzz.fuzzPackagedModule.FuzzProperties.Dictionary != nil {
		fuzz.fuzzPackagedModule.Dictionary = android.PathForModuleSrc(ctx, *fuzz.fuzzPackagedModule.FuzzProperties.Dictionary)
		if fuzz.fuzzPackagedModule.Dictionary.Ext() != ".dict" {
		ctx.PropertyErrorf("dictionary",
		"Fuzzer dictionary %q does not have '.dict' extension",
		fuzz.dictionary.String())
		fuzz.fuzzPackagedModule.Dictionary.String())
		}
		}

		if fuzz.Properties.Fuzz_config != nil {
		if fuzz.fuzzPackagedModule.FuzzProperties.Fuzz_config != nil {
		configPath := android.PathForModuleOut(ctx, "config").Join(ctx, "config.json")
		android.WriteFileRule(ctx, configPath, fuzz.Properties.Fuzz_config.String())
		fuzz.config = configPath
		android.WriteFileRule(ctx, configPath, fuzz.fuzzPackagedModule.FuzzProperties.Fuzz_config.String())
		fuzz.fuzzPackagedModule.Config = configPath
		}

		// Grab the list of required shared libraries.
		@@ -359,32 +354,20 @@ func NewFuzz(hod android.HostOrDeviceSupported) *Module {

		// Responsible for generating GNU Make rules that package fuzz targets into
		// their architecture & target/host specific zip file.
		type fuzzPackager struct {
		packages android.Paths
		type ccFuzzPackager struct {
		FuzzPackager
		sharedLibInstallStrings []string
		fuzzTargets map[string]bool
		}

		func fuzzPackagingFactory() android.Singleton {
		return &fuzzPackager{}
		}

		type fileToZip struct {
		SourceFilePath android.Path
		DestinationPathPrefix string
		}

		type archOs struct {
		hostOrTarget string
		arch string
		dir string
		return &ccFuzzPackager{}
		}

		func (s *fuzzPackager) GenerateBuildActions(ctx android.SingletonContext) {
		func (s *ccFuzzPackager) GenerateBuildActions(ctx android.SingletonContext) {
		// Map between each architecture + host/device combination, and the files that
		// need to be packaged (in the tuple of {source file, destination folder in
		// archive}).
		archDirs := make(map[archOs][]fileToZip)
		archDirs := make(map[ArchOs][]FileToZip)

		// Map tracking whether each shared library has an install rule to avoid duplicate install rules from
		// multiple fuzzers that depend on the same shared library.
		@@ -392,29 +375,21 @@ func (s *fuzzPackager) GenerateBuildActions(ctx android.SingletonContext) {

		// List of individual fuzz targets, so that 'make fuzz' also installs the targets
		// to the correct output directories as well.
		s.fuzzTargets = make(map[string]bool)
		s.FuzzTargets = make(map[string]bool)

		ctx.VisitAllModules(func(module android.Module) {
		// Discard non-fuzz targets.
		ccModule, ok := module.(*Module)
		if !ok {
		return
		}

		fuzzModule, ok := ccModule.compiler.(*fuzzBinary)
		if !ok {
		if !ok \|\| ccModule.Properties.PreventInstall {
		return
		}

		// Discard ramdisk + vendor_ramdisk + recovery modules, they're duplicates of
		// fuzz targets we're going to package anyway.
		if !ccModule.Enabled() \|\| ccModule.Properties.PreventInstall \|\|
		ccModule.InRamdisk() \|\| ccModule.InVendorRamdisk() \|\| ccModule.InRecovery() {
		// Discard non-fuzz targets.
		if ok := IsValid(ccModule.FuzzModule); !ok {
		return
		}

		// Discard modules that are in an unavailable namespace.
		if !ccModule.ExportedToMake() {
		fuzzModule, ok := ccModule.compiler.(*fuzzBinary)
		if !ok {
		return
		}

		@@ -425,42 +400,21 @@ func (s *fuzzPackager) GenerateBuildActions(ctx android.SingletonContext) {

		archString := ccModule.Arch().ArchType.String()
		archDir := android.PathForIntermediates(ctx, "fuzz", hostOrTargetString, archString)
		archOs := archOs{hostOrTarget: hostOrTargetString, arch: archString, dir: archDir.String()}
		archOs := ArchOs{HostOrTarget: hostOrTargetString, Arch: archString, Dir: archDir.String()}

		// Grab the list of required shared libraries.
		sharedLibraries := collectAllSharedDependencies(ctx, module)

		var files []fileToZip
		var files []FileToZip
		builder := android.NewRuleBuilder(pctx, ctx)

		// Package the corpora into a zipfile.
		if fuzzModule.corpus != nil {
		corpusZip := archDir.Join(ctx, module.Name()+"_seed_corpus.zip")
		command := builder.Command().BuiltTool("soong_zip").
		Flag("-j").
		FlagWithOutput("-o ", corpusZip)
		rspFile := corpusZip.ReplaceExtension(ctx, "rsp")
		command.FlagWithRspFileInputList("-r ", rspFile, fuzzModule.corpus)
		files = append(files, fileToZip{corpusZip, ""})
		}

		// Package the data into a zipfile.
		if fuzzModule.data != nil {
		dataZip := archDir.Join(ctx, module.Name()+"_data.zip")
		command := builder.Command().BuiltTool("soong_zip").
		FlagWithOutput("-o ", dataZip)
		for _, f := range fuzzModule.data {
		intermediateDir := strings.TrimSuffix(f.String(), f.Rel())
		command.FlagWithArg("-C ", intermediateDir)
		command.FlagWithInput("-f ", f)
		}
		files = append(files, fileToZip{dataZip, ""})
		}
		// Package the corpus, data, dict and config into a zipfile.
		files = s.PackageArtifacts(ctx, module, fuzzModule.fuzzPackagedModule, archDir, builder)

		// Find and mark all the transiently-dependent shared libraries for
		// packaging.
		for _, library := range sharedLibraries {
		files = append(files, fileToZip{library, "lib"})
		files = append(files, FileToZip{library, "lib"})

		// For each architecture-specific shared library dependency, we need to
		// install it to the output directory. Setup the install destination here,
		@@ -492,83 +446,20 @@ func (s *fuzzPackager) GenerateBuildActions(ctx android.SingletonContext) {
		}

		// The executable.
		files = append(files, fileToZip{ccModule.UnstrippedOutputFile(), ""})

		// The dictionary.
		if fuzzModule.dictionary != nil {
		files = append(files, fileToZip{fuzzModule.dictionary, ""})
		}

		// Additional fuzz config.
		if fuzzModule.config != nil {
		files = append(files, fileToZip{fuzzModule.config, ""})
		}

		fuzzZip := archDir.Join(ctx, module.Name()+".zip")
		command := builder.Command().BuiltTool("soong_zip").
		Flag("-j").
		FlagWithOutput("-o ", fuzzZip)
		for _, file := range files {
		if file.DestinationPathPrefix != "" {
		command.FlagWithArg("-P ", file.DestinationPathPrefix)
		} else {
		command.Flag("-P ''")
		}
		command.FlagWithInput("-f ", file.SourceFilePath)
		}

		builder.Build("create-"+fuzzZip.String(),
		"Package "+module.Name()+" for "+archString+"-"+hostOrTargetString)
		files = append(files, FileToZip{ccModule.UnstrippedOutputFile(), ""})

		// Don't add modules to 'make haiku' that are set to not be exported to the
		// fuzzing infrastructure.
		if config := fuzzModule.Properties.Fuzz_config; config != nil {
		if ccModule.Host() && !BoolDefault(config.Fuzz_on_haiku_host, true) {
		return
		} else if !BoolDefault(config.Fuzz_on_haiku_device, true) {
		archDirs[archOs], ok = s.BuildZipFile(ctx, module, fuzzModule.fuzzPackagedModule, files, builder, archDir, archString, hostOrTargetString, archOs, archDirs)
		if !ok {
		return
		}
		}

		s.fuzzTargets[module.Name()] = true
		archDirs[archOs] = append(archDirs[archOs], fileToZip{fuzzZip, ""})
		})

		var archOsList []archOs
		for archOs := range archDirs {
		archOsList = append(archOsList, archOs)
		}
		sort.Slice(archOsList, func(i, j int) bool { return archOsList[i].dir < archOsList[j].dir })

		for _, archOs := range archOsList {
		filesToZip := archDirs[archOs]
		arch := archOs.arch
		hostOrTarget := archOs.hostOrTarget
		builder := android.NewRuleBuilder(pctx, ctx)
		outputFile := android.PathForOutput(ctx, "fuzz-"+hostOrTarget+"-"+arch+".zip")
		s.packages = append(s.packages, outputFile)

		command := builder.Command().BuiltTool("soong_zip").
		Flag("-j").
		FlagWithOutput("-o ", outputFile).
		Flag("-L 0") // No need to try and re-compress the zipfiles.
		s.CreateFuzzPackage(ctx, archDirs, Cc)

		for _, fileToZip := range filesToZip {
		if fileToZip.DestinationPathPrefix != "" {
		command.FlagWithArg("-P ", fileToZip.DestinationPathPrefix)
		} else {
		command.Flag("-P ''")
		}
		command.FlagWithInput("-f ", fileToZip.SourceFilePath)
		}

		builder.Build("create-fuzz-package-"+arch+"-"+hostOrTarget,
		"Create fuzz target packages for "+arch+"-"+hostOrTarget)
		}
		}

		func (s *fuzzPackager) MakeVars(ctx android.MakeVarsContext) {
		packages := s.packages.Strings()
		func (s *ccFuzzPackager) MakeVars(ctx android.MakeVarsContext) {
		packages := s.Packages.Strings()
		sort.Strings(packages)
		sort.Strings(s.sharedLibInstallStrings)
		// TODO(mitchp): Migrate this to use MakeVarsContext::DistForGoal() when it's
		@@ -580,10 +471,5 @@ func (s *fuzzPackager) MakeVars(ctx android.MakeVarsContext) {
		strings.Join(s.sharedLibInstallStrings, " "))

		// Preallocate the slice of fuzz targets to minimise memory allocations.
		fuzzTargets := make([]string, 0, len(s.fuzzTargets))
		for target, _ := range s.fuzzTargets {
		fuzzTargets = append(fuzzTargets, target)
		}
		sort.Strings(fuzzTargets)
		ctx.Strict("ALL_FUZZ_TARGETS", strings.Join(fuzzTargets, " "))
		s.PreallocateSlice(ctx, "ALL_FUZZ_TARGETS")
		}

cc/fuzz_common.go

0 → 100644

+201 −0

Original line number	Diff line number	Diff line
		// Copyright 2021 Google Inc. All rights reserved.
		//
		// Licensed under the Apache License, Version 2.0 (the "License");
		// you may not use this file except in compliance with the License.
		// You may obtain a copy of the License at
		//
		// http://www.apache.org/licenses/LICENSE-2.0
		//
		// Unless required by applicable law or agreed to in writing, software
		// distributed under the License is distributed on an "AS IS" BASIS,
		// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
		// See the License for the specific language governing permissions and
		// limitations under the License.

		package cc

		// This file contains the common code for compiling C/C++ and Rust fuzzers for Android.

		import (
		"sort"
		"strings"

		"android/soong/android"
		)

		type Lang string

		const (
		Cc Lang = ""
		Rust Lang = "rust"
		)

		type FuzzModule struct {
		android.ModuleBase
		android.DefaultableModuleBase
		android.ApexModuleBase
		}

		type FuzzPackager struct {
		Packages android.Paths
		FuzzTargets map[string]bool
		}

		type FileToZip struct {
		SourceFilePath android.Path
		DestinationPathPrefix string
		}

		type ArchOs struct {
		HostOrTarget string
		Arch string
		Dir string
		}

		type FuzzPackagedModule struct {
		FuzzProperties FuzzProperties
		Dictionary android.Path
		Corpus android.Paths
		CorpusIntermediateDir android.Path
		Config android.Path
		Data android.Paths
		DataIntermediateDir android.Path
		}

		func IsValid(fuzzModule FuzzModule) bool {
		// Discard ramdisk + vendor_ramdisk + recovery modules, they're duplicates of
		// fuzz targets we're going to package anyway.
		if !fuzzModule.Enabled() \|\| fuzzModule.InRamdisk() \|\| fuzzModule.InVendorRamdisk() \|\| fuzzModule.InRecovery() {
		return false
		}

		// Discard modules that are in an unavailable namespace.
		if !fuzzModule.ExportedToMake() {
		return false
		}

		return true
		}

		func (s FuzzPackager) PackageArtifacts(ctx android.SingletonContext, module android.Module, fuzzModule FuzzPackagedModule, archDir android.OutputPath, builder android.RuleBuilder) []FileToZip {
		// Package the corpora into a zipfile.
		var files []FileToZip
		if fuzzModule.Corpus != nil {
		corpusZip := archDir.Join(ctx, module.Name()+"_seed_corpus.zip")
		command := builder.Command().BuiltTool("soong_zip").
		Flag("-j").
		FlagWithOutput("-o ", corpusZip)
		rspFile := corpusZip.ReplaceExtension(ctx, "rsp")
		command.FlagWithRspFileInputList("-r ", rspFile, fuzzModule.Corpus)
		files = append(files, FileToZip{corpusZip, ""})
		}

		// Package the data into a zipfile.
		if fuzzModule.Data != nil {
		dataZip := archDir.Join(ctx, module.Name()+"_data.zip")
		command := builder.Command().BuiltTool("soong_zip").
		FlagWithOutput("-o ", dataZip)
		for _, f := range fuzzModule.Data {
		intermediateDir := strings.TrimSuffix(f.String(), f.Rel())
		command.FlagWithArg("-C ", intermediateDir)
		command.FlagWithInput("-f ", f)
		}
		files = append(files, FileToZip{dataZip, ""})
		}

		// The dictionary.
		if fuzzModule.Dictionary != nil {
		files = append(files, FileToZip{fuzzModule.Dictionary, ""})
		}

		// Additional fuzz config.
		if fuzzModule.Config != nil {
		files = append(files, FileToZip{fuzzModule.Config, ""})
		}

		return files
		}

		func (s FuzzPackager) BuildZipFile(ctx android.SingletonContext, module android.Module, fuzzModule FuzzPackagedModule, files []FileToZip, builder android.RuleBuilder, archDir android.OutputPath, archString string, hostOrTargetString string, archOs ArchOs, archDirs map[ArchOs][]FileToZip) ([]FileToZip, bool) {
		fuzzZip := archDir.Join(ctx, module.Name()+".zip")

		command := builder.Command().BuiltTool("soong_zip").
		Flag("-j").
		FlagWithOutput("-o ", fuzzZip)

		for _, file := range files {
		if file.DestinationPathPrefix != "" {
		command.FlagWithArg("-P ", file.DestinationPathPrefix)
		} else {
		command.Flag("-P ''")
		}
		command.FlagWithInput("-f ", file.SourceFilePath)
		}

		builder.Build("create-"+fuzzZip.String(),
		"Package "+module.Name()+" for "+archString+"-"+hostOrTargetString)

		// Don't add modules to 'make haiku-rust' that are set to not be
		// exported to the fuzzing infrastructure.
		if config := fuzzModule.FuzzProperties.Fuzz_config; config != nil {
		if strings.Contains(hostOrTargetString, "host") && !BoolDefault(config.Fuzz_on_haiku_host, true) {
		return archDirs[archOs], false
		} else if !BoolDefault(config.Fuzz_on_haiku_device, true) {
		return archDirs[archOs], false
		}
		}

		s.FuzzTargets[module.Name()] = true
		archDirs[archOs] = append(archDirs[archOs], FileToZip{fuzzZip, ""})

		return archDirs[archOs], true
		}

		func (s *FuzzPackager) CreateFuzzPackage(ctx android.SingletonContext, archDirs map[ArchOs][]FileToZip, lang Lang) {
		var archOsList []ArchOs
		for archOs := range archDirs {
		archOsList = append(archOsList, archOs)
		}
		sort.Slice(archOsList, func(i, j int) bool { return archOsList[i].Dir < archOsList[j].Dir })

		for _, archOs := range archOsList {
		filesToZip := archDirs[archOs]
		arch := archOs.Arch
		hostOrTarget := archOs.HostOrTarget
		builder := android.NewRuleBuilder(pctx, ctx)
		zipFileName := "fuzz-" + hostOrTarget + "-" + arch + ".zip"
		if lang == Rust {
		zipFileName = "fuzz-rust-" + hostOrTarget + "-" + arch + ".zip"
		}
		outputFile := android.PathForOutput(ctx, zipFileName)

		s.Packages = append(s.Packages, outputFile)

		command := builder.Command().BuiltTool("soong_zip").
		Flag("-j").
		FlagWithOutput("-o ", outputFile).
		Flag("-L 0") // No need to try and re-compress the zipfiles.

		for _, fileToZip := range filesToZip {

		if fileToZip.DestinationPathPrefix != "" {
		command.FlagWithArg("-P ", fileToZip.DestinationPathPrefix)
		} else {
		command.Flag("-P ''")
		}
		command.FlagWithInput("-f ", fileToZip.SourceFilePath)

		}
		builder.Build("create-fuzz-package-"+arch+"-"+hostOrTarget,
		"Create fuzz target packages for "+arch+"-"+hostOrTarget)
		}
		}

		func (s *FuzzPackager) PreallocateSlice(ctx android.MakeVarsContext, targets string) {
		fuzzTargets := make([]string, 0, len(s.FuzzTargets))
		for target, _ := range s.FuzzTargets {
		fuzzTargets = append(fuzzTargets, target)
		}
		sort.Strings(fuzzTargets)
		ctx.Strict(targets, strings.Join(fuzzTargets, " "))
		}