Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 41ad881e authored by hamzeh's avatar hamzeh
Browse files

Refactoring fuzz packaging code 

Rust and cc fuzz packaging have common. This change is to put this
common methods and fields into fuzz_common.

Test: make haiku and make haiku-rust ran locally and verify corpus,
dict and config are packaged along with the executable.

Change-Id: I04bf535cd2597ed8b2bd3d8e6ea8c28f38bdc18b
parent 393bffee

cc/Android.bp

+1 −0
Original line number Diff line number Diff line
@@ -58,6 +58,7 @@ bootstrap_go_package { 
        "binary.go",

        "binary_sdk_member.go",

        "fuzz.go",

        "fuzz_common.go",

        "library.go",

        "library_headers.go",

        "library_sdk_member.go",

cc/androidmk.go

+8 −8
Original line number Diff line number Diff line
@@ -401,24 +401,24 @@ func (fuzz *fuzzBinary) AndroidMkEntries(ctx AndroidMkContext, entries *android. 
	ctx.subAndroidMk(entries, fuzz.binaryDecorator)



	var fuzzFiles []string

	for _, d := range fuzz.corpus {

	for _, d := range fuzz.fuzzPackagedModule.Corpus {

		fuzzFiles = append(fuzzFiles,

			filepath.Dir(fuzz.corpusIntermediateDir.String())+":corpus/"+d.Base())

			filepath.Dir(fuzz.fuzzPackagedModule.CorpusIntermediateDir.String())+":corpus/"+d.Base())

	}



	for _, d := range fuzz.data {

	for _, d := range fuzz.fuzzPackagedModule.Data {

		fuzzFiles = append(fuzzFiles,

			filepath.Dir(fuzz.dataIntermediateDir.String())+":data/"+d.Rel())

			filepath.Dir(fuzz.fuzzPackagedModule.DataIntermediateDir.String())+":data/"+d.Rel())

	}



	if fuzz.dictionary != nil {

	if fuzz.fuzzPackagedModule.Dictionary != nil {

		fuzzFiles = append(fuzzFiles,

			filepath.Dir(fuzz.dictionary.String())+":"+fuzz.dictionary.Base())

			filepath.Dir(fuzz.fuzzPackagedModule.Dictionary.String())+":"+fuzz.fuzzPackagedModule.Dictionary.Base())

	}



	if fuzz.config != nil {

	if fuzz.fuzzPackagedModule.Config != nil {

		fuzzFiles = append(fuzzFiles,

			filepath.Dir(fuzz.config.String())+":config.json")

			filepath.Dir(fuzz.fuzzPackagedModule.Config.String())+":config.json")

	}



	entries.ExtraEntries = append(entries.ExtraEntries, func(ctx android.AndroidMkExtraEntriesContext, entries *android.AndroidMkEntries) {

cc/cc.go

+3 −4
Original line number Diff line number Diff line
@@ -756,14 +756,13 @@ func IsTestPerSrcDepTag(depTag blueprint.DependencyTag) bool { 
// members of the cc.Module to this decorator. Thus, a cc_binary module has custom linker and

// installer logic.

type Module struct {

	android.ModuleBase

	android.DefaultableModuleBase

	android.ApexModuleBase

	FuzzModule



	android.SdkBase

	android.BazelModuleBase



	Properties       BaseProperties

	VendorProperties VendorProperties

	Properties       BaseProperties



	// initialize before calling Init

	hod      android.HostOrDeviceSupported

cc/fuzz.go

+41 −155
Original line number Diff line number Diff line
@@ -94,19 +94,14 @@ type fuzzBinary struct { 
	*binaryDecorator

	*baseCompiler



	Properties            FuzzProperties

	dictionary            android.Path

	corpus                android.Paths

	corpusIntermediateDir android.Path

	config                android.Path

	data                  android.Paths

	dataIntermediateDir   android.Path

	fuzzPackagedModule FuzzPackagedModule



	installedSharedDeps []string

}



func (fuzz *fuzzBinary) linkerProps() []interface{} {

	props := fuzz.binaryDecorator.linkerProps()

	props = append(props, &fuzz.Properties)

	props = append(props, &fuzz.fuzzPackagedModule.FuzzProperties)

	return props

}
@@ -257,41 +252,41 @@ func (fuzz *fuzzBinary) install(ctx ModuleContext, file android.Path) { 
		"fuzz", ctx.Target().Arch.ArchType.String(), ctx.ModuleName())

	fuzz.binaryDecorator.baseInstaller.install(ctx, file)



	fuzz.corpus = android.PathsForModuleSrc(ctx, fuzz.Properties.Corpus)

	fuzz.fuzzPackagedModule.Corpus = android.PathsForModuleSrc(ctx, fuzz.fuzzPackagedModule.FuzzProperties.Corpus)

	builder := android.NewRuleBuilder(pctx, ctx)

	intermediateDir := android.PathForModuleOut(ctx, "corpus")

	for _, entry := range fuzz.corpus {

	for _, entry := range fuzz.fuzzPackagedModule.Corpus {

		builder.Command().Text("cp").

			Input(entry).

			Output(intermediateDir.Join(ctx, entry.Base()))

	}

	builder.Build("copy_corpus", "copy corpus")

	fuzz.corpusIntermediateDir = intermediateDir

	fuzz.fuzzPackagedModule.CorpusIntermediateDir = intermediateDir



	fuzz.data = android.PathsForModuleSrc(ctx, fuzz.Properties.Data)

	fuzz.fuzzPackagedModule.Data = android.PathsForModuleSrc(ctx, fuzz.fuzzPackagedModule.FuzzProperties.Data)

	builder = android.NewRuleBuilder(pctx, ctx)

	intermediateDir = android.PathForModuleOut(ctx, "data")

	for _, entry := range fuzz.data {

	for _, entry := range fuzz.fuzzPackagedModule.Data {

		builder.Command().Text("cp").

			Input(entry).

			Output(intermediateDir.Join(ctx, entry.Rel()))

	}

	builder.Build("copy_data", "copy data")

	fuzz.dataIntermediateDir = intermediateDir

	fuzz.fuzzPackagedModule.DataIntermediateDir = intermediateDir



	if fuzz.Properties.Dictionary != nil {

		fuzz.dictionary = android.PathForModuleSrc(ctx, *fuzz.Properties.Dictionary)

		if fuzz.dictionary.Ext() != ".dict" {

	if fuzz.fuzzPackagedModule.FuzzProperties.Dictionary != nil {

		fuzz.fuzzPackagedModule.Dictionary = android.PathForModuleSrc(ctx, *fuzz.fuzzPackagedModule.FuzzProperties.Dictionary)

		if fuzz.fuzzPackagedModule.Dictionary.Ext() != ".dict" {

			ctx.PropertyErrorf("dictionary",

				"Fuzzer dictionary %q does not have '.dict' extension",

				fuzz.dictionary.String())

				fuzz.fuzzPackagedModule.Dictionary.String())

		}

	}



	if fuzz.Properties.Fuzz_config != nil {

	if fuzz.fuzzPackagedModule.FuzzProperties.Fuzz_config != nil {

		configPath := android.PathForModuleOut(ctx, "config").Join(ctx, "config.json")

		android.WriteFileRule(ctx, configPath, fuzz.Properties.Fuzz_config.String())

		fuzz.config = configPath

		android.WriteFileRule(ctx, configPath, fuzz.fuzzPackagedModule.FuzzProperties.Fuzz_config.String())

		fuzz.fuzzPackagedModule.Config = configPath

	}



	// Grab the list of required shared libraries.
@@ -359,32 +354,20 @@ func NewFuzz(hod android.HostOrDeviceSupported) *Module { 


// Responsible for generating GNU Make rules that package fuzz targets into

// their architecture & target/host specific zip file.

type fuzzPackager struct {

	packages                android.Paths

type ccFuzzPackager struct {

	FuzzPackager

	sharedLibInstallStrings []string

	fuzzTargets             map[string]bool

}



func fuzzPackagingFactory() android.Singleton {

	return &fuzzPackager{}

}



type fileToZip struct {

	SourceFilePath        android.Path

	DestinationPathPrefix string

}



type archOs struct {

	hostOrTarget string

	arch         string

	dir          string

	return &ccFuzzPackager{}

}



func (s *fuzzPackager) GenerateBuildActions(ctx android.SingletonContext) {

func (s *ccFuzzPackager) GenerateBuildActions(ctx android.SingletonContext) {

	// Map between each architecture + host/device combination, and the files that

	// need to be packaged (in the tuple of {source file, destination folder in

	// archive}).

	archDirs := make(map[archOs][]fileToZip)

	archDirs := make(map[ArchOs][]FileToZip)



	// Map tracking whether each shared library has an install rule to avoid duplicate install rules from

	// multiple fuzzers that depend on the same shared library.
@@ -392,29 +375,21 @@ func (s *fuzzPackager) GenerateBuildActions(ctx android.SingletonContext) { 


	// List of individual fuzz targets, so that 'make fuzz' also installs the targets

	// to the correct output directories as well.

	s.fuzzTargets = make(map[string]bool)

	s.FuzzTargets = make(map[string]bool)



	ctx.VisitAllModules(func(module android.Module) {

		// Discard non-fuzz targets.

		ccModule, ok := module.(*Module)

		if !ok {

			return

		}



		fuzzModule, ok := ccModule.compiler.(*fuzzBinary)

		if !ok {

		if !ok || ccModule.Properties.PreventInstall {

			return

		}



		// Discard ramdisk + vendor_ramdisk + recovery modules, they're duplicates of

		// fuzz targets we're going to package anyway.

		if !ccModule.Enabled() || ccModule.Properties.PreventInstall ||

			ccModule.InRamdisk() || ccModule.InVendorRamdisk() || ccModule.InRecovery() {

		// Discard non-fuzz targets.

		if ok := IsValid(ccModule.FuzzModule); !ok {

			return

		}



		// Discard modules that are in an unavailable namespace.

		if !ccModule.ExportedToMake() {

		fuzzModule, ok := ccModule.compiler.(*fuzzBinary)

		if !ok {

			return

		}
@@ -425,42 +400,21 @@ func (s *fuzzPackager) GenerateBuildActions(ctx android.SingletonContext) { 


		archString := ccModule.Arch().ArchType.String()

		archDir := android.PathForIntermediates(ctx, "fuzz", hostOrTargetString, archString)

		archOs := archOs{hostOrTarget: hostOrTargetString, arch: archString, dir: archDir.String()}

		archOs := ArchOs{HostOrTarget: hostOrTargetString, Arch: archString, Dir: archDir.String()}



		// Grab the list of required shared libraries.

		sharedLibraries := collectAllSharedDependencies(ctx, module)



		var files []fileToZip

		var files []FileToZip

		builder := android.NewRuleBuilder(pctx, ctx)



		// Package the corpora into a zipfile.

		if fuzzModule.corpus != nil {

			corpusZip := archDir.Join(ctx, module.Name()+"_seed_corpus.zip")

			command := builder.Command().BuiltTool("soong_zip").

				Flag("-j").

				FlagWithOutput("-o ", corpusZip)

			rspFile := corpusZip.ReplaceExtension(ctx, "rsp")

			command.FlagWithRspFileInputList("-r ", rspFile, fuzzModule.corpus)

			files = append(files, fileToZip{corpusZip, ""})

		}



		// Package the data into a zipfile.

		if fuzzModule.data != nil {

			dataZip := archDir.Join(ctx, module.Name()+"_data.zip")

			command := builder.Command().BuiltTool("soong_zip").

				FlagWithOutput("-o ", dataZip)

			for _, f := range fuzzModule.data {

				intermediateDir := strings.TrimSuffix(f.String(), f.Rel())

				command.FlagWithArg("-C ", intermediateDir)

				command.FlagWithInput("-f ", f)

			}

			files = append(files, fileToZip{dataZip, ""})

		}

		// Package the corpus, data, dict and config into a zipfile.

		files = s.PackageArtifacts(ctx, module, fuzzModule.fuzzPackagedModule, archDir, builder)



		// Find and mark all the transiently-dependent shared libraries for

		// packaging.

		for _, library := range sharedLibraries {

			files = append(files, fileToZip{library, "lib"})

			files = append(files, FileToZip{library, "lib"})



			// For each architecture-specific shared library dependency, we need to

			// install it to the output directory. Setup the install destination here,
@@ -492,83 +446,20 @@ func (s *fuzzPackager) GenerateBuildActions(ctx android.SingletonContext) { 
		}



		// The executable.

		files = append(files, fileToZip{ccModule.UnstrippedOutputFile(), ""})



		// The dictionary.

		if fuzzModule.dictionary != nil {

			files = append(files, fileToZip{fuzzModule.dictionary, ""})

		}



		// Additional fuzz config.

		if fuzzModule.config != nil {

			files = append(files, fileToZip{fuzzModule.config, ""})

		}



		fuzzZip := archDir.Join(ctx, module.Name()+".zip")

		command := builder.Command().BuiltTool("soong_zip").

			Flag("-j").

			FlagWithOutput("-o ", fuzzZip)

		for _, file := range files {

			if file.DestinationPathPrefix != "" {

				command.FlagWithArg("-P ", file.DestinationPathPrefix)

			} else {

				command.Flag("-P ''")

			}

			command.FlagWithInput("-f ", file.SourceFilePath)

		}



		builder.Build("create-"+fuzzZip.String(),

			"Package "+module.Name()+" for "+archString+"-"+hostOrTargetString)

		files = append(files, FileToZip{ccModule.UnstrippedOutputFile(), ""})



		// Don't add modules to 'make haiku' that are set to not be exported to the

		// fuzzing infrastructure.

		if config := fuzzModule.Properties.Fuzz_config; config != nil {

			if ccModule.Host() && !BoolDefault(config.Fuzz_on_haiku_host, true) {

				return

			} else if !BoolDefault(config.Fuzz_on_haiku_device, true) {

		archDirs[archOs], ok = s.BuildZipFile(ctx, module, fuzzModule.fuzzPackagedModule, files, builder, archDir, archString, hostOrTargetString, archOs, archDirs)

		if !ok {

			return

		}

		}



		s.fuzzTargets[module.Name()] = true

		archDirs[archOs] = append(archDirs[archOs], fileToZip{fuzzZip, ""})

	})



	var archOsList []archOs

	for archOs := range archDirs {

		archOsList = append(archOsList, archOs)

	}

	sort.Slice(archOsList, func(i, j int) bool { return archOsList[i].dir < archOsList[j].dir })



	for _, archOs := range archOsList {

		filesToZip := archDirs[archOs]

		arch := archOs.arch

		hostOrTarget := archOs.hostOrTarget

		builder := android.NewRuleBuilder(pctx, ctx)

		outputFile := android.PathForOutput(ctx, "fuzz-"+hostOrTarget+"-"+arch+".zip")

		s.packages = append(s.packages, outputFile)



		command := builder.Command().BuiltTool("soong_zip").

			Flag("-j").

			FlagWithOutput("-o ", outputFile).

			Flag("-L 0") // No need to try and re-compress the zipfiles.

	s.CreateFuzzPackage(ctx, archDirs, Cc)



		for _, fileToZip := range filesToZip {

			if fileToZip.DestinationPathPrefix != "" {

				command.FlagWithArg("-P ", fileToZip.DestinationPathPrefix)

			} else {

				command.Flag("-P ''")

			}

			command.FlagWithInput("-f ", fileToZip.SourceFilePath)

}



		builder.Build("create-fuzz-package-"+arch+"-"+hostOrTarget,

			"Create fuzz target packages for "+arch+"-"+hostOrTarget)

	}

}



func (s *fuzzPackager) MakeVars(ctx android.MakeVarsContext) {

	packages := s.packages.Strings()

func (s *ccFuzzPackager) MakeVars(ctx android.MakeVarsContext) {

	packages := s.Packages.Strings()

	sort.Strings(packages)

	sort.Strings(s.sharedLibInstallStrings)

	// TODO(mitchp): Migrate this to use MakeVarsContext::DistForGoal() when it's
@@ -580,10 +471,5 @@ func (s *fuzzPackager) MakeVars(ctx android.MakeVarsContext) { 
		strings.Join(s.sharedLibInstallStrings, " "))



	// Preallocate the slice of fuzz targets to minimise memory allocations.

	fuzzTargets := make([]string, 0, len(s.fuzzTargets))

	for target, _ := range s.fuzzTargets {

		fuzzTargets = append(fuzzTargets, target)

	}

	sort.Strings(fuzzTargets)

	ctx.Strict("ALL_FUZZ_TARGETS", strings.Join(fuzzTargets, " "))

	s.PreallocateSlice(ctx, "ALL_FUZZ_TARGETS")

}

cc/fuzz_common.go

0 → 100644
+201 −0
Original line number Diff line number Diff line 
// Copyright 2021 Google Inc. All rights reserved.

//

// Licensed under the Apache License, Version 2.0 (the "License");

// you may not use this file except in compliance with the License.

// You may obtain a copy of the License at

//

//     http://www.apache.org/licenses/LICENSE-2.0

//

// Unless required by applicable law or agreed to in writing, software

// distributed under the License is distributed on an "AS IS" BASIS,

// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

// See the License for the specific language governing permissions and

// limitations under the License.



package cc



// This file contains the common code for compiling C/C++ and Rust fuzzers for Android.



import (

	"sort"

	"strings"



	"android/soong/android"

)



type Lang string



const (

	Cc   Lang = ""

	Rust Lang = "rust"

)



type FuzzModule struct {

	android.ModuleBase

	android.DefaultableModuleBase

	android.ApexModuleBase

}



type FuzzPackager struct {

	Packages    android.Paths

	FuzzTargets map[string]bool

}



type FileToZip struct {

	SourceFilePath        android.Path

	DestinationPathPrefix string

}



type ArchOs struct {

	HostOrTarget string

	Arch         string

	Dir          string

}



type FuzzPackagedModule struct {

	FuzzProperties        FuzzProperties

	Dictionary            android.Path

	Corpus                android.Paths

	CorpusIntermediateDir android.Path

	Config                android.Path

	Data                  android.Paths

	DataIntermediateDir   android.Path

}



func IsValid(fuzzModule FuzzModule) bool {

	// Discard ramdisk + vendor_ramdisk + recovery modules, they're duplicates of

	// fuzz targets we're going to package anyway.

	if !fuzzModule.Enabled() || fuzzModule.InRamdisk() || fuzzModule.InVendorRamdisk() || fuzzModule.InRecovery() {

		return false

	}



	// Discard modules that are in an unavailable namespace.

	if !fuzzModule.ExportedToMake() {

		return false

	}



	return true

}



func (s *FuzzPackager) PackageArtifacts(ctx android.SingletonContext, module android.Module, fuzzModule FuzzPackagedModule, archDir android.OutputPath, builder *android.RuleBuilder) []FileToZip {

	// Package the corpora into a zipfile.

	var files []FileToZip

	if fuzzModule.Corpus != nil {

		corpusZip := archDir.Join(ctx, module.Name()+"_seed_corpus.zip")

		command := builder.Command().BuiltTool("soong_zip").

			Flag("-j").

			FlagWithOutput("-o ", corpusZip)

		rspFile := corpusZip.ReplaceExtension(ctx, "rsp")

		command.FlagWithRspFileInputList("-r ", rspFile, fuzzModule.Corpus)

		files = append(files, FileToZip{corpusZip, ""})

	}



	// Package the data into a zipfile.

	if fuzzModule.Data != nil {

		dataZip := archDir.Join(ctx, module.Name()+"_data.zip")

		command := builder.Command().BuiltTool("soong_zip").

			FlagWithOutput("-o ", dataZip)

		for _, f := range fuzzModule.Data {

			intermediateDir := strings.TrimSuffix(f.String(), f.Rel())

			command.FlagWithArg("-C ", intermediateDir)

			command.FlagWithInput("-f ", f)

		}

		files = append(files, FileToZip{dataZip, ""})

	}



	// The dictionary.

	if fuzzModule.Dictionary != nil {

		files = append(files, FileToZip{fuzzModule.Dictionary, ""})

	}



	// Additional fuzz config.

	if fuzzModule.Config != nil {

		files = append(files, FileToZip{fuzzModule.Config, ""})

	}



	return files

}



func (s *FuzzPackager) BuildZipFile(ctx android.SingletonContext, module android.Module, fuzzModule FuzzPackagedModule, files []FileToZip, builder *android.RuleBuilder, archDir android.OutputPath, archString string, hostOrTargetString string, archOs ArchOs, archDirs map[ArchOs][]FileToZip) ([]FileToZip, bool) {

	fuzzZip := archDir.Join(ctx, module.Name()+".zip")



	command := builder.Command().BuiltTool("soong_zip").

		Flag("-j").

		FlagWithOutput("-o ", fuzzZip)



	for _, file := range files {

		if file.DestinationPathPrefix != "" {

			command.FlagWithArg("-P ", file.DestinationPathPrefix)

		} else {

			command.Flag("-P ''")

		}

		command.FlagWithInput("-f ", file.SourceFilePath)

	}



	builder.Build("create-"+fuzzZip.String(),

		"Package "+module.Name()+" for "+archString+"-"+hostOrTargetString)



	// Don't add modules to 'make haiku-rust' that are set to not be

	// exported to the fuzzing infrastructure.

	if config := fuzzModule.FuzzProperties.Fuzz_config; config != nil {

		if strings.Contains(hostOrTargetString, "host") && !BoolDefault(config.Fuzz_on_haiku_host, true) {

			return archDirs[archOs], false

		} else if !BoolDefault(config.Fuzz_on_haiku_device, true) {

			return archDirs[archOs], false

		}

	}



	s.FuzzTargets[module.Name()] = true

	archDirs[archOs] = append(archDirs[archOs], FileToZip{fuzzZip, ""})



	return archDirs[archOs], true

}



func (s *FuzzPackager) CreateFuzzPackage(ctx android.SingletonContext, archDirs map[ArchOs][]FileToZip, lang Lang) {

	var archOsList []ArchOs

	for archOs := range archDirs {

		archOsList = append(archOsList, archOs)

	}

	sort.Slice(archOsList, func(i, j int) bool { return archOsList[i].Dir < archOsList[j].Dir })



	for _, archOs := range archOsList {

		filesToZip := archDirs[archOs]

		arch := archOs.Arch

		hostOrTarget := archOs.HostOrTarget

		builder := android.NewRuleBuilder(pctx, ctx)

		zipFileName := "fuzz-" + hostOrTarget + "-" + arch + ".zip"

		if lang == Rust {

			zipFileName = "fuzz-rust-" + hostOrTarget + "-" + arch + ".zip"

		}

		outputFile := android.PathForOutput(ctx, zipFileName)



		s.Packages = append(s.Packages, outputFile)



		command := builder.Command().BuiltTool("soong_zip").

			Flag("-j").

			FlagWithOutput("-o ", outputFile).

			Flag("-L 0") // No need to try and re-compress the zipfiles.



		for _, fileToZip := range filesToZip {



			if fileToZip.DestinationPathPrefix != "" {

				command.FlagWithArg("-P ", fileToZip.DestinationPathPrefix)

			} else {

				command.Flag("-P ''")

			}

			command.FlagWithInput("-f ", fileToZip.SourceFilePath)



		}

		builder.Build("create-fuzz-package-"+arch+"-"+hostOrTarget,

			"Create fuzz target packages for "+arch+"-"+hostOrTarget)

	}

}



func (s *FuzzPackager) PreallocateSlice(ctx android.MakeVarsContext, targets string) {

	fuzzTargets := make([]string, 0, len(s.FuzzTargets))

	for target, _ := range s.FuzzTargets {

		fuzzTargets = append(fuzzTargets, target)

	}

	sort.Strings(fuzzTargets)

	ctx.Strict(targets, strings.Join(fuzzTargets, " "))

}