Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 2a4549ec authored by Sasha Smundak's avatar Sasha Smundak
Browse files

Support source code cross-referencing for C++ and Java 

Use Kythe (https://kythe.io) to build cross reference for the Android
source code. ~generate the input for it during the build. This is done
on demand: if XREF_CORPUS environment variable is set, build emits a
Ninja rule to generate Kythe input for each compilation rule. It
also emits two consolidation rules (`xref_cxx` and `xref_java`),
that depend on all Kythe input generation rules for C++ and Java.

The value of the XREF_CORPUS environment variable is recorded in the
generated files and thus passed to Kythe. For the AOSP master branch it is
`android.googlesource.com/platform/superproject`, so the command to build
all input for Kythe on that branch is:
```
XREF_CORPUS=android.googlesource.com/platform/superproject m xref_cxx xref_java
```

Each Kythe input generation rule generates a single file with .kzip
extension. Individual .kzip files have a lot of common information, so
there will be a post-build consolidation step run to combine them.
The consolidated .kzip file is then passed to Kythe backend.

The tools to generate .kzip files are provided by Kythe (it calls them
'extractors'). We are going to build them in toolbuilding branches
(clang-tools and build-tools) and check them in as binaries into master
and other PDK branches:
For C++,  `prebuilts/clang-tools/linux-x86/bin/cxx_extractor`
for Java, `prebuilts/build-tools/common/framework/javac_extractor.jar`

Bug: 121267023
Test: 1) When XREF_CORPUS is set, build generates Ninja rules to create
.kzip files; 2) When XREF_CORPUS is set, building
`xref_cxx`/`xref_java` creates .kzip files; 3) Unless XREF_CORPUS is
set, build generates the same Ninja rules as before

Change-Id: If957b35d7abc82dbfbb3665980e7c34afe7c789e
parent e515886a

android/config.go

+8 −0
Original line number Diff line number Diff line
@@ -756,6 +756,14 @@ func (c *config) RunErrorProne() bool { 
	return c.IsEnvTrue("RUN_ERROR_PRONE")

}



func (c *config) XrefCorpusName() string {

	return c.Getenv("XREF_CORPUS")

}



func (c *config) EmitXrefRules() bool {

	return c.XrefCorpusName() != ""

}



// Returns true if -source 1.9 -target 1.9 is being passed to javac

func (c *config) TargetOpenJDK9() bool {

	return c.targetOpenJDK9

cc/builder.go

+38 −0
Original line number Diff line number Diff line
@@ -221,6 +221,17 @@ var ( 
			Rspfile:        "$out.rsp",

			RspfileContent: "$in",

		})



	_ = pctx.SourcePathVariable("cxxExtractor",

		"prebuilts/clang-tools/${config.HostPrebuiltTag}/bin/cxx_extractor")

	_ = pctx.VariableFunc("kytheCorpus",

		func(ctx android.PackageVarContext) string { return ctx.Config().XrefCorpusName() })

	kytheExtract = pctx.StaticRule("kythe",

		blueprint.RuleParams{

			Command:     "rm -f $out && KYTHE_CORPUS=${kytheCorpus} KYTHE_OUTPUT_FILE=$out $cxxExtractor $cFlags $in ",

			CommandDeps: []string{"$cxxExtractor"},

		},

		"cFlags")

)



func init() {
@@ -257,6 +268,7 @@ type builderFlags struct { 
	tidy            bool

	coverage        bool

	sAbiDump        bool

	emitXrefs       bool



	systemIncludeFlags string
@@ -281,6 +293,7 @@ type Objects struct { 
	tidyFiles     android.Paths

	coverageFiles android.Paths

	sAbiDumpFiles android.Paths

	kytheFiles    android.Paths

}



func (a Objects) Copy() Objects {
@@ -289,6 +302,7 @@ func (a Objects) Copy() Objects { 
		tidyFiles:     append(android.Paths{}, a.tidyFiles...),

		coverageFiles: append(android.Paths{}, a.coverageFiles...),

		sAbiDumpFiles: append(android.Paths{}, a.sAbiDumpFiles...),

		kytheFiles:    append(android.Paths{}, a.kytheFiles...),

	}

}
@@ -298,6 +312,7 @@ func (a Objects) Append(b Objects) Objects { 
		tidyFiles:     append(a.tidyFiles, b.tidyFiles...),

		coverageFiles: append(a.coverageFiles, b.coverageFiles...),

		sAbiDumpFiles: append(a.sAbiDumpFiles, b.sAbiDumpFiles...),

		kytheFiles:    append(a.kytheFiles, b.kytheFiles...),

	}

}
@@ -314,6 +329,10 @@ func TransformSourceToObj(ctx android.ModuleContext, subdir string, srcFiles and 
	if flags.coverage {

		coverageFiles = make(android.Paths, 0, len(srcFiles))

	}

	var kytheFiles android.Paths

	if flags.emitXrefs {

		kytheFiles = make(android.Paths, 0, len(srcFiles))

	}



	commonFlags := strings.Join([]string{

		flags.globalFlags,
@@ -401,6 +420,7 @@ func TransformSourceToObj(ctx android.ModuleContext, subdir string, srcFiles and 
		coverage := flags.coverage

		dump := flags.sAbiDump

		rule := cc

		emitXref := flags.emitXrefs



		switch srcFile.Ext() {

		case ".s":
@@ -412,6 +432,7 @@ func TransformSourceToObj(ctx android.ModuleContext, subdir string, srcFiles and 
			tidy = false

			coverage = false

			dump = false

			emitXref = false

		case ".c":

			ccCmd = "clang"

			moduleCflags = cflags
@@ -450,6 +471,22 @@ func TransformSourceToObj(ctx android.ModuleContext, subdir string, srcFiles and 
			},

		})



		if emitXref {

			kytheFile := android.ObjPathWithExt(ctx, subdir, srcFile, "kzip")

			ctx.Build(pctx, android.BuildParams{

				Rule:        kytheExtract,

				Description: "Xref C++ extractor " + srcFile.Rel(),

				Output:      kytheFile,

				Input:       srcFile,

				Implicits:   cFlagsDeps,

				OrderOnly:   pathDeps,

				Args: map[string]string{

					"cFlags": moduleCflags,

				},

			})

			kytheFiles = append(kytheFiles, kytheFile)

		}



		if tidy {

			tidyFile := android.ObjPathWithExt(ctx, subdir, srcFile, "tidy")

			tidyFiles = append(tidyFiles, tidyFile)
@@ -493,6 +530,7 @@ func TransformSourceToObj(ctx android.ModuleContext, subdir string, srcFiles and 
		tidyFiles:     tidyFiles,

		coverageFiles: coverageFiles,

		sAbiDumpFiles: sAbiDumpFiles,

		kytheFiles:    kytheFiles,

	}

}

cc/cc.go

+39 −0
Original line number Diff line number Diff line
@@ -77,6 +77,7 @@ func init() { 
		ctx.TopDown("double_loadable", checkDoubleLoadableLibraries).Parallel()

	})



	android.RegisterSingletonType("kythe_extract_all", kytheExtractAllFactory)

	pctx.Import("android/soong/cc/config")

}
@@ -162,6 +163,7 @@ type Flags struct { 
	Tidy      bool

	Coverage  bool

	SAbiDump  bool

	EmitXrefs bool // If true, generate Ninja rules to generate emitXrefs input files for Kythe



	RequiredInstructionSet string

	DynamicLinker          string
@@ -346,6 +348,10 @@ type dependencyTag struct { 
	explicitlyVersioned bool

}



type xref interface {

	XrefCcFiles() android.Paths

}



var (

	sharedDepTag          = dependencyTag{name: "shared", library: true}

	sharedExportDepTag    = dependencyTag{name: "shared", library: true, reexportFlags: true}
@@ -427,6 +433,8 @@ type Module struct { 
	staticVariant *Module



	makeLinkType string

	// Kythe (source file indexer) paths for this compilation module

	kytheFiles android.Paths

}



func (c *Module) OutputFile() android.OptionalPath {
@@ -657,6 +665,10 @@ func installToBootstrap(name string, config android.Config) bool { 
	return isBionic(name)

}



func (c *Module) XrefCcFiles() android.Paths {

	return c.kytheFiles

}



type baseModuleContext struct {

	android.BaseModuleContext

	moduleContextImpl
@@ -995,6 +1007,7 @@ func (c *Module) GenerateAndroidBuildActions(actx android.ModuleContext) { 


	flags := Flags{

		Toolchain: c.toolchain(ctx),

		EmitXrefs: ctx.Config().EmitXrefRules(),

	}

	if c.compiler != nil {

		flags = c.compiler.compilerFlags(ctx, flags, deps)
@@ -1060,6 +1073,7 @@ func (c *Module) GenerateAndroidBuildActions(actx android.ModuleContext) { 
		if ctx.Failed() {

			return

		}

		c.kytheFiles = objs.kytheFiles

	}



	if c.linker != nil {
@@ -2366,6 +2380,31 @@ func getCurrentNdkPrebuiltVersion(ctx DepsContext) string { 
	return ctx.Config().PlatformSdkVersion()

}



func kytheExtractAllFactory() android.Singleton {

	return &kytheExtractAllSingleton{}

}



type kytheExtractAllSingleton struct {

}



func (ks *kytheExtractAllSingleton) GenerateBuildActions(ctx android.SingletonContext) {

	var xrefTargets android.Paths

	ctx.VisitAllModules(func(module android.Module) {

		if ccModule, ok := module.(xref); ok {

			xrefTargets = append(xrefTargets, ccModule.XrefCcFiles()...)

		}

	})

	// TODO(asmundak): Perhaps emit a rule to output a warning if there were no xrefTargets

	if len(xrefTargets) > 0 {

		ctx.Build(pctx, android.BuildParams{

			Rule:   blueprint.Phony,

			Output: android.PathForPhony(ctx, "xref_cxx"),

			Inputs: xrefTargets,

			//Default: true,

		})

	}

}



var Bool = proptools.Bool

var BoolDefault = proptools.BoolDefault

var BoolPtr = proptools.BoolPtr

cc/util.go

+1 −0
Original line number Diff line number Diff line
@@ -74,6 +74,7 @@ func flagsToBuilderFlags(in Flags) builderFlags { 
		coverage:        in.Coverage,

		tidy:            in.Tidy,

		sAbiDump:        in.SAbiDump,

		emitXrefs:       in.EmitXrefs,



		systemIncludeFlags: strings.Join(in.SystemIncludeFlags, " "),

java/builder.go

+86 −0
Original line number Diff line number Diff line
@@ -62,6 +62,37 @@ var ( 
		"javacFlags", "bootClasspath", "classpath", "processorpath", "processor", "srcJars", "srcJarDir",

		"outDir", "annoDir", "javaVersion")



	_ = pctx.VariableFunc("kytheCorpus",

		func(ctx android.PackageVarContext) string { return ctx.Config().XrefCorpusName() })

	// Run it with -add-opens=java.base/java.nio=ALL-UNNAMED to avoid JDK9's warning about

	// "Illegal reflective access by com.google.protobuf.Utf8$UnsafeProcessor ...

	// to field java.nio.Buffer.address"

	kytheExtract = pctx.AndroidStaticRule("kythe",

		blueprint.RuleParams{

			Command: `${config.ZipSyncCmd} -d $srcJarDir ` +

				`-l $srcJarDir/list -f "*.java" $srcJars && ` +

				`( [ ! -s $srcJarDir/list -a ! -s $out.rsp ] || ` +

				`KYTHE_ROOT_DIRECTORY=. KYTHE_OUTPUT_FILE=$out ` +

				`KYTHE_CORPUS=${kytheCorpus} ` +

				`${config.SoongJavacWrapper} ${config.JavaCmd} ` +

				`--add-opens=java.base/java.nio=ALL-UNNAMED ` +

				`-jar ${config.JavaKytheExtractorJar} ` +

				`${config.JavacHeapFlags} ${config.CommonJdkFlags} ` +

				`$processorpath $processor $javacFlags $bootClasspath $classpath ` +

				`-source $javaVersion -target $javaVersion ` +

				`-d $outDir -s $annoDir @$out.rsp @$srcJarDir/list)`,

			CommandDeps: []string{

				"${config.JavaCmd}",

				"${config.JavaKytheExtractorJar}",

				"${config.ZipSyncCmd}",

			},

			CommandOrderOnly: []string{"${config.SoongJavacWrapper}"},

			Rspfile:          "$out.rsp",

			RspfileContent:   "$in",

		},

		"javacFlags", "bootClasspath", "classpath", "processorpath", "processor", "srcJars", "srcJarDir",

		"outDir", "annoDir", "javaVersion")



	turbine = pctx.AndroidStaticRule("turbine",

		blueprint.RuleParams{

			Command: `rm -rf "$outDir" && mkdir -p "$outDir" && ` +
@@ -196,6 +227,61 @@ func RunErrorProne(ctx android.ModuleContext, outputFile android.WritablePath, 
		"errorprone", "errorprone")

}



// Emits the rule to generate Xref input file (.kzip file) for the given set of source files and source jars

// to compile with given set of builder flags, etc.

func emitXrefRule(ctx android.ModuleContext, xrefFile android.WritablePath,

	srcFiles, srcJars android.Paths,

	flags javaBuilderFlags, deps android.Paths,

	intermediatesDir string) {



	deps = append(deps, srcJars...)



	var bootClasspath string

	if flags.javaVersion == "1.9" {

		var systemModuleDeps android.Paths

		bootClasspath, systemModuleDeps = flags.systemModules.FormJavaSystemModulesPath(ctx.Device())

		deps = append(deps, systemModuleDeps...)

	} else {

		deps = append(deps, flags.bootClasspath...)

		if len(flags.bootClasspath) == 0 && ctx.Device() {

			// explicitly specify -bootclasspath "" if the bootclasspath is empty to

			// ensure java does not fall back to the default bootclasspath.

			bootClasspath = `-bootclasspath ""`

		} else {

			bootClasspath = flags.bootClasspath.FormJavaClassPath("-bootclasspath")

		}

	}



	deps = append(deps, flags.classpath...)

	deps = append(deps, flags.processorPath...)



	processor := "-proc:none"

	if flags.processor != "" {

		processor = "-processor " + flags.processor

	}



	ctx.Build(pctx,

		android.BuildParams{

			Rule:        kytheExtract,

			Description: "Xref Java extractor",

			Output:      xrefFile,

			Inputs:      srcFiles,

			Implicits:   deps,

			Args: map[string]string{

				"annoDir":       android.PathForModuleOut(ctx, intermediatesDir, "anno").String(),

				"bootClasspath": bootClasspath,

				"classpath":     flags.classpath.FormJavaClassPath("-classpath"),

				"javacFlags":    flags.javacFlags,

				"javaVersion":   flags.javaVersion,

				"outDir":        android.PathForModuleOut(ctx, "javac", "classes.xref").String(),

				"processorpath": flags.processorPath.FormJavaClassPath("-processorpath"),

				"processor":     processor,

				"srcJarDir":     android.PathForModuleOut(ctx, intermediatesDir, "srcjars.xref").String(),

				"srcJars":       strings.Join(srcJars.Strings(), " "),

			},

		})

}



func TransformJavaToHeaderClasses(ctx android.ModuleContext, outputFile android.WritablePath,

	srcFiles, srcJars android.Paths, flags javaBuilderFlags) {