Merge "[hoststubgen] Reusable TextFileFilterPolicyParser" into main (4637294f) · Commits · e / os / android_frameworks_base

ravenwood/tools/hoststubgen/scripts/dump-jar

+1 −1

Original line number	Diff line number	Diff line
		@@ -89,7 +89,7 @@ filter_output() {
		# - Some other transient lines
		# - Sometimes the javap shows mysterious warnings, so remove them too.
		#
		# `/PATTERN-1/,/PATTERN-1/{//!d}` is a trick to delete lines between two patterns, without
		# `/PATTERN-1/,/PATTERN-2/{//!d}` is a trick to delete lines between two patterns, without
		# the start and the end lines.
		sed -e 's/#[0-9][0-9]*/#x/g' \
		-e 's/^$ $[0-9][0-9]:/\1x:/' \

ravenwood/tools/hoststubgen/src/com/android/hoststubgen/HostStubGen.kt

+4 −4

Original line number	Diff line number	Diff line
		@@ -27,7 +27,7 @@ import com.android.hoststubgen.filters.ImplicitOutputFilter
		import com.android.hoststubgen.filters.KeepNativeFilter
		import com.android.hoststubgen.filters.OutputFilter
		import com.android.hoststubgen.filters.SanitizationFilter
		import com.android.hoststubgen.filters.TextFileFilterPolicyParser
		import com.android.hoststubgen.filters.TextFileFilterPolicyBuilder
		import com.android.hoststubgen.filters.printAsTextPolicy
		import com.android.hoststubgen.utils.ClassFilter
		import com.android.hoststubgen.visitors.BaseAdapter
		@@ -179,9 +179,9 @@ class HostStubGen(val options: HostStubGenOptions) {
		// Next, "text based" filter, which allows to override polices without touching
		// the target code.
		if (options.policyOverrideFiles.isNotEmpty()) {
		val parser = TextFileFilterPolicyParser(allClasses, filter)
		options.policyOverrideFiles.forEach(parser::parse)
		filter = parser.createOutputFilter()
		val builder = TextFileFilterPolicyBuilder(allClasses, filter)
		options.policyOverrideFiles.forEach(builder::parse)
		filter = builder.createOutputFilter()
		}

		// Apply the implicit filter.

ravenwood/tools/hoststubgen/src/com/android/hoststubgen/filters/TextFileFilterPolicyParser.kt

+305 −68

Original line number	Diff line number	Diff line
		@@ -23,10 +23,12 @@ import com.android.hoststubgen.asm.toJvmClassName
		import com.android.hoststubgen.log
		import com.android.hoststubgen.normalizeTextLine
		import com.android.hoststubgen.whitespaceRegex
		import java.io.File
		import org.objectweb.asm.tree.ClassNode
		import java.io.BufferedReader
		import java.io.FileReader
		import java.io.PrintWriter
		import java.io.Reader
		import java.util.regex.Pattern
		import org.objectweb.asm.tree.ClassNode

		/**
		* Print a class node as a "keep" policy.
		@@ -48,7 +50,7 @@ fun printAsTextPolicy(pw: PrintWriter, cn: ClassNode) {

		private const val FILTER_REASON = "file-override"

		private enum class SpecialClass {
		enum class SpecialClass {
		NotSpecial,
		Aidl,
		FeatureFlags,
		@@ -56,10 +58,58 @@ private enum class SpecialClass {
		RFile,
		}

		class TextFileFilterPolicyParser(
		/**
		* This receives [TextFileFilterPolicyBuilder] parsing result.
		*/
		interface PolicyFileProcessor {
		/** "package" directive. */
		fun onPackage(name: String, policy: FilterPolicyWithReason)

		/** "rename" directive. */
		fun onRename(pattern: Pattern, prefix: String)

		/** "class" directive. */
		fun onSimpleClassStart(className: String)
		fun onSimpleClassPolicy(className: String, policy: FilterPolicyWithReason)
		fun onSimpleClassEnd(className: String)

		fun onSubClassPolicy(superClassName: String, policy: FilterPolicyWithReason)
		fun onRedirectionClass(fromClassName: String, toClassName: String)
		fun onClassLoadHook(className: String, callback: String)
		fun onSpecialClassPolicy(type: SpecialClass, policy: FilterPolicyWithReason)

		/** "field" directive. */
		fun onField(className: String, fieldName: String, policy: FilterPolicyWithReason)

		/** "method" directive. */
		fun onSimpleMethodPolicy(
		className: String,
		methodName: String,
		methodDesc: String,
		policy: FilterPolicyWithReason,
		)
		fun onMethodInClassReplace(
		className: String,
		methodName: String,
		methodDesc: String,
		targetName: String,
		policy: FilterPolicyWithReason,
		)
		fun onMethodOutClassReplace(
		className: String,
		methodName: String,
		methodDesc: String,
		replaceSpec: TextFilePolicyMethodReplaceFilter.MethodCallReplaceSpec,
		policy: FilterPolicyWithReason,
		)
		}

		class TextFileFilterPolicyBuilder(
		private val classes: ClassNodes,
		fallback: OutputFilter
		) {
		private val parser = TextFileFilterPolicyParser()

		private val subclassFilter = SubclassFilter(classes, fallback)
		private val packageFilter = PackageFilter(subclassFilter)
		private val imf = InMemoryOutputFilter(classes, packageFilter)
		@@ -71,30 +121,19 @@ class TextFileFilterPolicyParser(
		private val methodReplaceSpec =
		mutableListOf<TextFilePolicyMethodReplaceFilter.MethodCallReplaceSpec>()

		private lateinit var currentClassName: String

		/**
		* Read a given "policy" file and return as an [OutputFilter]
		* Parse a given policy file. This method can be called multiple times to read from
		* multiple files. To get the resulting filter, use [createOutputFilter]
		*/
		fun parse(file: String) {
		log.i("Loading offloaded annotations from $file ...")
		log.withIndent {
		var lineNo = 0
		try {
		File(file).forEachLine {
		lineNo++
		val line = normalizeTextLine(it)
		if (line.isEmpty()) {
		return@forEachLine // skip empty lines.
		}
		parseLine(line)
		}
		} catch (e: ParseException) {
		throw e.withSourceInfo(file, lineNo)
		}
		}
		// We may parse multiple files, but we reuse the same parser, because the parser
		// will make sure there'll be no dupplicating "special class" policies.
		parser.parse(FileReader(file), file, Processor())
		}

		/**
		* Generate the resulting [OutputFilter].
		*/
		fun createOutputFilter(): OutputFilter {
		var ret: OutputFilter = imf
		if (typeRenameSpec.isNotEmpty()) {
		@@ -112,14 +151,200 @@ class TextFileFilterPolicyParser(
		return ret
		}

		private inner class Processor : PolicyFileProcessor {
		override fun onPackage(name: String, policy: FilterPolicyWithReason) {
		packageFilter.addPolicy(name, policy)
		}

		override fun onRename(pattern: Pattern, prefix: String) {
		typeRenameSpec += TextFilePolicyRemapperFilter.TypeRenameSpec(
		pattern, prefix
		)
		}

		override fun onSimpleClassStart(className: String) {
		}

		override fun onSimpleClassEnd(className: String) {
		}

		override fun onSimpleClassPolicy(className: String, policy: FilterPolicyWithReason) {
		imf.setPolicyForClass(className, policy)
		}

		override fun onSubClassPolicy(
		superClassName: String,
		policy: FilterPolicyWithReason,
		) {
		log.i("class extends $superClassName")
		subclassFilter.addPolicy( superClassName, policy)
		}

		override fun onRedirectionClass(fromClassName: String, toClassName: String) {
		imf.setRedirectionClass(fromClassName, toClassName)
		}

		override fun onClassLoadHook(className: String, callback: String) {
		imf.setClassLoadHook(className, callback)
		}

		override fun onSpecialClassPolicy(
		type: SpecialClass,
		policy: FilterPolicyWithReason,
		) {
		log.i("class special $type $policy")
		when (type) {
		SpecialClass.NotSpecial -> {} // Shouldn't happen

		SpecialClass.Aidl -> {
		aidlPolicy = policy
		}

		SpecialClass.FeatureFlags -> {
		featureFlagsPolicy = policy
		}

		SpecialClass.Sysprops -> {
		syspropsPolicy = policy
		}

		SpecialClass.RFile -> {
		rFilePolicy = policy
		}
		}
		}

		override fun onField(className: String, fieldName: String, policy: FilterPolicyWithReason) {
		imf.setPolicyForField(className, fieldName, policy)
		}

		override fun onSimpleMethodPolicy(
		className: String,
		methodName: String,
		methodDesc: String,
		policy: FilterPolicyWithReason,
		) {
		imf.setPolicyForMethod(className, methodName, methodDesc, policy)
		}

		override fun onMethodInClassReplace(
		className: String,
		methodName: String,
		methodDesc: String,
		targetName: String,
		policy: FilterPolicyWithReason,
		) {
		imf.setPolicyForMethod(className, methodName, methodDesc, policy)

		// Make sure to keep the target method.
		imf.setPolicyForMethod(
		className,
		targetName,
		methodDesc,
		FilterPolicy.Keep.withReason(FILTER_REASON)
		)
		// Set up the rename.
		imf.setRenameTo(className, targetName, methodDesc, methodName)
		}

		override fun onMethodOutClassReplace(
		className: String,
		methodName: String,
		methodDesc: String,
		replaceSpec: TextFilePolicyMethodReplaceFilter.MethodCallReplaceSpec,
		policy: FilterPolicyWithReason,
		) {
		imf.setPolicyForMethod(className, methodName, methodDesc, policy)
		methodReplaceSpec.add(replaceSpec)
		}
		}
		}

		/**
		* Parses a filer policy text file.
		*/
		class TextFileFilterPolicyParser {
		private lateinit var processor: PolicyFileProcessor
		private var currentClassName: String? = null

		private var aidlPolicy: FilterPolicyWithReason? = null
		private var featureFlagsPolicy: FilterPolicyWithReason? = null
		private var syspropsPolicy: FilterPolicyWithReason? = null
		private var rFilePolicy: FilterPolicyWithReason? = null

		/** Name of the file that's currently being processed. */
		var filename: String? = null
		private set

		/** 1-based line number in the current file */
		var lineNumber = -1
		private set

		/**
		* Parse a given "policy" file.
		*/
		fun parse(reader: Reader, inputName: String, processor: PolicyFileProcessor) {
		filename = inputName

		log.i("Parsing text policy file $inputName ...")
		this.processor = processor
		BufferedReader(reader).use { rd ->
		lineNumber = 0
		try {
		while (true) {
		var line = rd.readLine()
		if (line == null) {
		break
		}
		lineNumber++
		line = normalizeTextLine(line) // Remove comment and trim.
		if (line.isEmpty()) {
		continue
		}
		parseLine(line)
		}
		finishLastClass()
		} catch (e: ParseException) {
		throw e.withSourceInfo(inputName, lineNumber)
		}
		}
		}

		private fun finishLastClass() {
		currentClassName?.let { className ->
		processor.onSimpleClassEnd(className)
		currentClassName = null
		}
		}

		private fun ensureInClass(directive: String): String {
		return currentClassName ?:
		throw ParseException("Directive '$directive' must follow a 'class' directive")
		}

		private fun parseLine(line: String) {
		val fields = line.split(whitespaceRegex).toTypedArray()
		when (fields[0].lowercase()) {
		"p", "package" -> parsePackage(fields)
		"c", "class" -> parseClass(fields)
		"f", "field" -> parseField(fields)
		"m", "method" -> parseMethod(fields)
		"r", "rename" -> parseRename(fields)
		"p", "package" -> {
		finishLastClass()
		parsePackage(fields)
		}
		"c", "class" -> {
		finishLastClass()
		parseClass(fields)
		}
		"f", "field" -> {
		ensureInClass("field")
		parseField(fields)
		}
		"m", "method" -> {
		ensureInClass("method")
		parseMethod(fields)
		}
		"r", "rename" -> {
		finishLastClass()
		parseRename(fields)
		}
		else -> throw ParseException("Unknown directive \"${fields[0]}\"")
		}
		}
		@@ -184,20 +409,20 @@ class TextFileFilterPolicyParser(
		if (!policy.isUsableWithClasses) {
		throw ParseException("Package can't have policy '$policy'")
		}
		packageFilter.addPolicy(name, policy.withReason(FILTER_REASON))
		processor.onPackage(name, policy.withReason(FILTER_REASON))
		}

		private fun parseClass(fields: Array<String>) {
		if (fields.size < 3) {
		throw ParseException("Class ('c') expects 2 fields.")
		}
		currentClassName = fields[1]
		val className = fields[1]

		// superClass is set when the class name starts with a "*".
		val superClass = resolveExtendingClass(currentClassName)
		val superClass = resolveExtendingClass(className)

		// :aidl, etc?
		val classType = resolveSpecialClass(currentClassName)
		val classType = resolveSpecialClass(className)

		if (fields[2].startsWith("!")) {
		if (classType != SpecialClass.NotSpecial) {
		@@ -208,7 +433,8 @@ class TextFileFilterPolicyParser(
		}
		// It's a redirection class.
		val toClass = fields[2].substring(1)
		imf.setRedirectionClass(currentClassName, toClass)

		processor.onRedirectionClass(className, toClass)
		} else if (fields[2].startsWith("~")) {
		if (classType != SpecialClass.NotSpecial) {
		// We could support it, but not needed at least for now.
		@@ -218,7 +444,8 @@ class TextFileFilterPolicyParser(
		}
		// It's a class-load hook
		val callback = fields[2].substring(1)
		imf.setClassLoadHook(currentClassName, callback)

		processor.onClassLoadHook(className, callback)
		} else {
		val policy = parsePolicy(fields[2])
		if (!policy.isUsableWithClasses) {
		@@ -229,26 +456,27 @@ class TextFileFilterPolicyParser(
		SpecialClass.NotSpecial -> {
		// TODO: Duplicate check, etc
		if (superClass == null) {
		imf.setPolicyForClass(
		currentClassName, policy.withReason(FILTER_REASON)
		)
		currentClassName = className
		processor.onSimpleClassStart(className)
		processor.onSimpleClassPolicy(className, policy.withReason(FILTER_REASON))
		} else {
		subclassFilter.addPolicy(
		processor.onSubClassPolicy(
		superClass,
		policy.withReason("extends $superClass")
		policy.withReason("extends $superClass"),
		)
		}
		}

		SpecialClass.Aidl -> {
		if (aidlPolicy != null) {
		throw ParseException(
		"Policy for AIDL classes already defined"
		)
		}
		aidlPolicy = policy.withReason(
		val p = policy.withReason(
		"$FILTER_REASON (special-class AIDL)"
		)
		processor.onSpecialClassPolicy(classType, p)
		aidlPolicy = p
		}

		SpecialClass.FeatureFlags -> {
		@@ -257,9 +485,11 @@ class TextFileFilterPolicyParser(
		"Policy for feature flags already defined"
		)
		}
		featureFlagsPolicy = policy.withReason(
		val p = policy.withReason(
		"$FILTER_REASON (special-class feature flags)"
		)
		processor.onSpecialClassPolicy(classType, p)
		featureFlagsPolicy = p
		}

		SpecialClass.Sysprops -> {
		@@ -268,9 +498,11 @@ class TextFileFilterPolicyParser(
		"Policy for sysprops already defined"
		)
		}
		syspropsPolicy = policy.withReason(
		val p = policy.withReason(
		"$FILTER_REASON (special-class sysprops)"
		)
		processor.onSpecialClassPolicy(classType, p)
		syspropsPolicy = p
		}

		SpecialClass.RFile -> {
		@@ -279,9 +511,11 @@ class TextFileFilterPolicyParser(
		"Policy for R file already defined"
		)
		}
		rFilePolicy = policy.withReason(
		val p = policy.withReason(
		"$FILTER_REASON (special-class R file)"
		)
		processor.onSpecialClassPolicy(classType, p)
		rFilePolicy = p
		}
		}
		}
		@@ -296,17 +530,16 @@ class TextFileFilterPolicyParser(
		if (!policy.isUsableWithFields) {
		throw ParseException("Field can't have policy '$policy'")
		}
		require(this::currentClassName.isInitialized)

		// TODO: Duplicate check, etc
		imf.setPolicyForField(currentClassName, name, policy.withReason(FILTER_REASON))
		processor.onField(currentClassName!!, name, policy.withReason(FILTER_REASON))
		}

		private fun parseMethod(fields: Array<String>) {
		if (fields.size < 3 \|\| fields.size > 4) {
		throw ParseException("Method ('m') expects 3 or 4 fields.")
		}
		val name = fields[1]
		val methodName = fields[1]
		val signature: String
		val policyStr: String
		if (fields.size <= 3) {
		@@ -323,44 +556,48 @@ class TextFileFilterPolicyParser(
		throw ParseException("Method can't have policy '$policy'")
		}

		require(this::currentClassName.isInitialized)
		val className = currentClassName!!

		imf.setPolicyForMethod(
		currentClassName, name, signature,
		policy.withReason(FILTER_REASON)
		)
		if (policy == FilterPolicy.Substitute) {
		val fromName = policyStr.substring(1)
		val policyWithReason = policy.withReason(FILTER_REASON)
		if (policy != FilterPolicy.Substitute) {
		processor.onSimpleMethodPolicy(className, methodName, signature, policyWithReason)
		} else {
		val targetName = policyStr.substring(1)

		if (fromName == name) {
		if (targetName == methodName) {
		throw ParseException(
		"Substitution must have a different name"
		)
		}

		// Set the policy for the "from" method.
		imf.setPolicyForMethod(
		currentClassName, fromName, signature,
		FilterPolicy.Keep.withReason(FILTER_REASON)
		)

		val classAndMethod = splitWithLastPeriod(fromName)
		val classAndMethod = splitWithLastPeriod(targetName)
		if (classAndMethod != null) {
		// If the substitution target contains a ".", then
		// it's a method call redirect.
		methodReplaceSpec.add(
		TextFilePolicyMethodReplaceFilter.MethodCallReplaceSpec(
		currentClassName.toJvmClassName(),
		name,
		val spec = TextFilePolicyMethodReplaceFilter.MethodCallReplaceSpec(
		currentClassName!!.toJvmClassName(),
		methodName,
		signature,
		classAndMethod.first.toJvmClassName(),
		classAndMethod.second,
		)
		processor.onMethodOutClassReplace(
		className,
		methodName,
		signature,
		spec,
		policyWithReason,
		)
		} else {
		// It's an in-class replace.
		// ("@RavenwoodReplace" equivalent)
		imf.setRenameTo(currentClassName, fromName, signature, name)
		processor.onMethodInClassReplace(
		className,
		methodName,
		signature,
		targetName,
		policyWithReason,
		)
		}
		}
		}
		@@ -378,7 +615,7 @@ class TextFileFilterPolicyParser(
		// applied. (Which is needed for services.jar)
		val prefix = fields[2].trimStart('/')

		typeRenameSpec += TextFilePolicyRemapperFilter.TypeRenameSpec(
		processor.onRename(
		pattern, prefix
		)
		}

ravenwood/tools/hoststubgen/test-tiny-framework/diff-and-update-golden.sh

+4 −3

Original line number	Diff line number	Diff line
		@@ -34,10 +34,11 @@ source "${0%/*}"/../common.sh

		SCRIPT_NAME="${0##*/}"

		GOLDEN_DIR=golden-output
		GOLDEN_DIR=${GOLDEN_DIR:-golden-output}
		mkdir -p $GOLDEN_DIR

		DIFF_CMD=${DIFF:-diff -u --ignore-blank-lines --ignore-space-change}
		# TODO(b/388562869) We shouldn't need `--ignore-matching-lines`, but the golden files may not have the "Constant pool:" lines.
		DIFF_CMD=${DIFF_CMD:-diff -u --ignore-blank-lines --ignore-space-change --ignore-matching-lines='^$Constant.pool:\\|{$$'}

		update=0
		three_way=0
		@@ -62,7 +63,7 @@ done
		shift $(($OPTIND - 1))

		# Build the dump files, which are the input of this test.
		run m dump-jar tiny-framework-dump-test
		run ${BUILD_CMD:=m} dump-jar tiny-framework-dump-test


		# Get the path to the generate text files. (not the golden files.)

ravenwood/tools/hoststubgen/test-tiny-framework/tiny-framework-dump-test.py

+4 −1

Original line number	Diff line number	Diff line
		@@ -28,8 +28,11 @@ GOLDEN_DIRS = [

		# Run diff.
		def run_diff(file1, file2):
		# TODO(b/388562869) We shouldn't need `--ignore-matching-lines`, but the golden files may not have the "Constant pool:" lines.
		command = ['diff', '-u', '--ignore-blank-lines',
		'--ignore-space-change', file1, file2]
		'--ignore-space-change',
		'--ignore-matching-lines=^$Constant.pool:\\|{$$',
		file1, file2]
		print(' '.join(command))
		result = subprocess.run(command, stderr=sys.stdout)