protolog: improve build time of frameworks/base/services (fbcd32e5) · Commits · e / os / android_frameworks_base

tools/protologtool/src/com/android/protolog/tool/ProtoLogTool.kt

+65 −31

Original line number	Original line	Diff line number	Diff line
	@@ -18,11 +18,14 @@ package com.android.protolog.tool

	import com.android.protolog.tool.CommandOptions.Companion.USAGE		import com.android.protolog.tool.CommandOptions.Companion.USAGE
	import com.github.javaparser.ParseProblemException		import com.github.javaparser.ParseProblemException
			import com.github.javaparser.ParserConfiguration
	import com.github.javaparser.StaticJavaParser		import com.github.javaparser.StaticJavaParser
	import com.github.javaparser.ast.CompilationUnit		import com.github.javaparser.ast.CompilationUnit
	import java.io.File		import java.io.File
	import java.io.FileInputStream		import java.io.FileInputStream
	import java.io.FileOutputStream		import java.io.FileOutputStream
			import java.util.concurrent.ExecutorService
			import java.util.concurrent.Executors
	import java.util.jar.JarOutputStream		import java.util.jar.JarOutputStream
	import java.util.zip.ZipEntry		import java.util.zip.ZipEntry
	import kotlin.system.exitProcess		import kotlin.system.exitProcess
	@@ -45,16 +48,18 @@ object ProtoLogTool {
	val outJar = JarOutputStream(out)		val outJar = JarOutputStream(out)
	val processor = ProtoLogCallProcessor(command.protoLogClassNameArg,		val processor = ProtoLogCallProcessor(command.protoLogClassNameArg,
	command.protoLogGroupsClassNameArg, groups)		command.protoLogGroupsClassNameArg, groups)
	val transformer = SourceTransformer(command.protoLogImplClassNameArg, processor)

	command.javaSourceArgs.forEach { path ->		val executor = newThreadPool()

			command.javaSourceArgs.map { path ->
			executor.submitCallable {
			val transformer = SourceTransformer(command.protoLogImplClassNameArg, processor)
	val file = File(path)		val file = File(path)
	val text = file.readText()		val text = file.readText()
	val newPath = path
	val outSrc = try {		val outSrc = try {
	val code = tryParse(text, path)		val code = tryParse(text, path)
	if (containsProtoLogText(text, command.protoLogClassNameArg)) {		if (containsProtoLogText(text, command.protoLogClassNameArg)) {
	transformer.processClass(text, newPath, code)		transformer.processClass(text, path, code)
	} else {		} else {
	text		text
	}		}
	@@ -64,11 +69,17 @@ object ProtoLogTool {
	println("\n${ex.message}\n")		println("\n${ex.message}\n")
	text		text
	}		}
	outJar.putNextEntry(ZipEntry(newPath))		path to outSrc
			}
			}.map { future ->
			val (path, outSrc) = future.get()
			outJar.putNextEntry(ZipEntry(path))
	outJar.write(outSrc.toByteArray())		outJar.write(outSrc.toByteArray())
	outJar.closeEntry()		outJar.closeEntry()
	}		}

			executor.shutdown()

	outJar.close()		outJar.close()
	out.close()		out.close()
	}		}
	@@ -92,7 +103,11 @@ object ProtoLogTool {
	val processor = ProtoLogCallProcessor(command.protoLogClassNameArg,		val processor = ProtoLogCallProcessor(command.protoLogClassNameArg,
	command.protoLogGroupsClassNameArg, groups)		command.protoLogGroupsClassNameArg, groups)
	val builder = ViewerConfigBuilder(processor)		val builder = ViewerConfigBuilder(processor)
	command.javaSourceArgs.forEach { path ->
			val executor = newThreadPool()

			command.javaSourceArgs.map { path ->
			executor.submitCallable {
	val file = File(path)		val file = File(path)
	val text = file.readText()		val text = file.readText()
	if (containsProtoLogText(text, command.protoLogClassNameArg)) {		if (containsProtoLogText(text, command.protoLogClassNameArg)) {
	@@ -101,14 +116,23 @@ object ProtoLogTool {
	val pack = if (code.packageDeclaration.isPresent) code.packageDeclaration		val pack = if (code.packageDeclaration.isPresent) code.packageDeclaration
	.get().nameAsString else ""		.get().nameAsString else ""
	val newPath = pack.replace('.', '/') + '/' + file.name		val newPath = pack.replace('.', '/') + '/' + file.name
	builder.processClass(code, newPath)		builder.findLogCalls(code, newPath)
	} catch (ex: ParsingException) {		} catch (ex: ParsingException) {
	// If we cannot parse this file, skip it (and log why). Compilation will fail		// If we cannot parse this file, skip it (and log why). Compilation will fail
	// in a subsequent build step.		// in a subsequent build step.
	println("\n${ex.message}\n")		println("\n${ex.message}\n")
			null
			}
			} else {
			null
	}		}
	}		}
			}.forEach { future ->
			builder.addLogCalls(future.get() ?: return@forEach)
	}		}

			executor.shutdown()

	val out = FileOutputStream(command.viewerConfigJsonArg)		val out = FileOutputStream(command.viewerConfigJsonArg)
	out.write(builder.build().toByteArray())		out.write(builder.build().toByteArray())
	out.close()		out.close()
	@@ -122,6 +146,11 @@ object ProtoLogTool {

	@JvmStatic		@JvmStatic
	fun main(args: Array<String>) {		fun main(args: Array<String>) {
			StaticJavaParser.setConfiguration(ParserConfiguration().apply {
			setLanguageLevel(ParserConfiguration.LanguageLevel.RAW)
			setAttributeComments(false)
			})

	try {		try {
	val command = CommandOptions(args)		val command = CommandOptions(args)
	when (command.command) {		when (command.command) {
	@@ -138,3 +167,8 @@ object ProtoLogTool {
	}		}
	}		}
	}		}

			private fun <T> ExecutorService.submitCallable(f: () -> T) = submit(f)

			private fun newThreadPool() = Executors.newFixedThreadPool(
			Runtime.getRuntime().availableProcessors())

tools/protologtool/src/com/android/protolog/tool/SourceTransformer.kt

+2 −4

Original line number	Original line	Diff line number	Diff line
	@@ -42,7 +42,6 @@ import com.github.javaparser.ast.type.PrimitiveType
	import com.github.javaparser.ast.type.Type		import com.github.javaparser.ast.type.Type
	import com.github.javaparser.printer.PrettyPrinter		import com.github.javaparser.printer.PrettyPrinter
	import com.github.javaparser.printer.PrettyPrinterConfiguration		import com.github.javaparser.printer.PrettyPrinterConfiguration
	import com.github.javaparser.printer.lexicalpreservation.LexicalPreservingPrinter

	class SourceTransformer(		class SourceTransformer(
	protoLogImplClassName: String,		protoLogImplClassName: String,
	@@ -135,7 +134,8 @@ class SourceTransformer(
	// Inline the new statement.		// Inline the new statement.
	val printedIfStmt = inlinePrinter.print(ifStmt)		val printedIfStmt = inlinePrinter.print(ifStmt)
	// Append blank lines to preserve line numbering in file (to allow debugging)		// Append blank lines to preserve line numbering in file (to allow debugging)
	val newLines = LexicalPreservingPrinter.print(parentStmt).count { c -> c == '\n' }		val parentRange = parentStmt.range.get()
			val newLines = parentRange.end.line - parentRange.begin.line
	val newStmt = printedIfStmt.substringBeforeLast('}') + ("\n".repeat(newLines)) + '}'		val newStmt = printedIfStmt.substringBeforeLast('}') + ("\n".repeat(newLines)) + '}'
	// pre-workaround code, see explanation below		// pre-workaround code, see explanation below
	/*		/*
	@@ -224,9 +224,7 @@ class SourceTransformer(
	fileName = path		fileName = path
	processedCode = code.split('\n').toMutableList()		processedCode = code.split('\n').toMutableList()
	offsets = IntArray(processedCode.size)		offsets = IntArray(processedCode.size)
	LexicalPreservingPrinter.setup(compilationUnit)
	protoLogCallProcessor.process(compilationUnit, this, fileName)		protoLogCallProcessor.process(compilationUnit, this, fileName)
	// return LexicalPreservingPrinter.print(compilationUnit)
	return processedCode.joinToString("\n")		return processedCode.joinToString("\n")
	}		}
	}		}

tools/protologtool/src/com/android/protolog/tool/ViewerConfigBuilder.kt

+35 −20

Original line number	Original line	Diff line number	Diff line
	@@ -23,39 +23,52 @@ import com.github.javaparser.ast.expr.MethodCallExpr
	import java.io.StringWriter		import java.io.StringWriter

	class ViewerConfigBuilder(		class ViewerConfigBuilder(
	private val protoLogCallVisitor: ProtoLogCallProcessor		private val processor: ProtoLogCallProcessor
	) : ProtoLogCallVisitor {
	override fun processCall(
	call: MethodCallExpr,
	messageString: String,
	level: LogLevel,
	group: LogGroup
	) {		) {
			private fun addLogCall(logCall: LogCall, context: ParsingContext) {
			val group = logCall.logGroup
			val messageString = logCall.messageString
	if (group.enabled) {		if (group.enabled) {
	val position = fileName		val key = logCall.key()
	val key = CodeUtils.hash(position, messageString, level, group)
	if (statements.containsKey(key)) {		if (statements.containsKey(key)) {
	if (statements[key] != LogCall(messageString, level, group, position)) {		if (statements[key] != logCall) {
	throw HashCollisionException(		throw HashCollisionException(
	"Please modify the log message \"$messageString\" " +		"Please modify the log message \"$messageString\" " +
	"or \"${statements[key]}\" - their hashes are equal.",		"or \"${statements[key]}\" - their hashes are equal.", context)
	ParsingContext(fileName, call))
	}		}
	} else {		} else {
	groups.add(group)		groups.add(group)
	statements[key] = LogCall(messageString, level, group, position)		statements[key] = logCall
	call.range.isPresent
	}		}
	}		}
	}		}

	private val statements: MutableMap<Int, LogCall> = mutableMapOf()		private val statements: MutableMap<Int, LogCall> = mutableMapOf()
	private val groups: MutableSet<LogGroup> = mutableSetOf()		private val groups: MutableSet<LogGroup> = mutableSetOf()
	private var fileName: String = ""

	fun processClass(unit: CompilationUnit, fileName: String) {		fun findLogCalls(unit: CompilationUnit, fileName: String): List<Pair<LogCall, ParsingContext>> {
	this.fileName = fileName		val calls = mutableListOf<Pair<LogCall, ParsingContext>>()
	protoLogCallVisitor.process(unit, this, fileName)		val visitor = object : ProtoLogCallVisitor {
			override fun processCall(
			call: MethodCallExpr,
			messageString: String,
			level: LogLevel,
			group: LogGroup
			) {
			val logCall = LogCall(messageString, level, group, fileName)
			val context = ParsingContext(fileName, call)
			calls.add(logCall to context)
			}
			}
			processor.process(unit, visitor, fileName)

			return calls
			}

			fun addLogCalls(calls: List<Pair<LogCall, ParsingContext>>) {
			calls.forEach { (logCall, context) ->
			addLogCall(logCall, context)
			}
	}		}

	fun build(): String {		fun build(): String {
	@@ -101,5 +114,7 @@ class ViewerConfigBuilder(
	val logLevel: LogLevel,		val logLevel: LogLevel,
	val logGroup: LogGroup,		val logGroup: LogGroup,
	val position: String		val position: String
	)		) {
			fun key() = CodeUtils.hash(position, messageString, logLevel, logGroup)
			}
	}		}

tools/protologtool/tests/com/android/protolog/tool/ViewerConfigBuilderTest.kt

+23 −59

Original line number	Original line	Diff line number	Diff line
	@@ -17,8 +17,7 @@
	package com.android.protolog.tool		package com.android.protolog.tool

	import com.android.json.stream.JsonReader		import com.android.json.stream.JsonReader
	import com.github.javaparser.ast.CompilationUnit		import com.android.protolog.tool.ViewerConfigBuilder.LogCall
	import com.github.javaparser.ast.expr.MethodCallExpr
	import org.junit.Assert.assertEquals		import org.junit.Assert.assertEquals
	import org.junit.Test		import org.junit.Test
	import org.mockito.Mockito		import org.mockito.Mockito
	@@ -34,14 +33,12 @@ class ViewerConfigBuilderTest {
	private val GROUP1 = LogGroup("TEST_GROUP", true, true, TAG1)		private val GROUP1 = LogGroup("TEST_GROUP", true, true, TAG1)
	private val GROUP2 = LogGroup("DEBUG_GROUP", true, true, TAG2)		private val GROUP2 = LogGroup("DEBUG_GROUP", true, true, TAG2)
	private val GROUP3 = LogGroup("DEBUG_GROUP", true, true, TAG2)		private val GROUP3 = LogGroup("DEBUG_GROUP", true, true, TAG2)
			private val GROUP_DISABLED = LogGroup("DEBUG_GROUP", false, true, TAG2)
			private val GROUP_TEXT_DISABLED = LogGroup("DEBUG_GROUP", true, false, TAG2)
	private const val PATH = "/tmp/test.java"		private const val PATH = "/tmp/test.java"
	}		}

	private val processor: ProtoLogCallProcessor = Mockito.mock(ProtoLogCallProcessor::class.java)		private val configBuilder = ViewerConfigBuilder(Mockito.mock(ProtoLogCallProcessor::class.java))
	private val configBuilder = ViewerConfigBuilder(processor)
	private val dummyCompilationUnit = CompilationUnit()

	private fun <T> any(type: Class<T>): T = Mockito.any<T>(type)

	private fun parseConfig(json: String): Map<Int, ViewerConfigParser.ConfigEntry> {		private fun parseConfig(json: String): Map<Int, ViewerConfigParser.ConfigEntry> {
	return ViewerConfigParser().parseConfig(JsonReader(StringReader(json)))		return ViewerConfigParser().parseConfig(JsonReader(StringReader(json)))
	@@ -49,22 +46,10 @@ class ViewerConfigBuilderTest {

	@Test		@Test
	fun processClass() {		fun processClass() {
	Mockito.`when`(processor.process(any(CompilationUnit::class.java),		configBuilder.addLogCalls(listOf(
	any(ProtoLogCallVisitor::class.java), any(String::class.java)))		LogCall(TEST1.messageString, LogLevel.INFO, GROUP1, PATH),
	.thenAnswer { invocation ->		LogCall(TEST2.messageString, LogLevel.DEBUG, GROUP2, PATH),
	val visitor = invocation.arguments[1] as ProtoLogCallVisitor		LogCall(TEST3.messageString, LogLevel.ERROR, GROUP3, PATH)).withContext())

	visitor.processCall(MethodCallExpr(), TEST1.messageString, LogLevel.INFO,
	GROUP1)
	visitor.processCall(MethodCallExpr(), TEST2.messageString, LogLevel.DEBUG,
	GROUP2)
	visitor.processCall(MethodCallExpr(), TEST3.messageString, LogLevel.ERROR,
	GROUP3)

	invocation.arguments[0] as CompilationUnit
	}

	configBuilder.processClass(dummyCompilationUnit, PATH)

	val parsedConfig = parseConfig(configBuilder.build())		val parsedConfig = parseConfig(configBuilder.build())
	assertEquals(3, parsedConfig.size)		assertEquals(3, parsedConfig.size)
	@@ -78,22 +63,10 @@ class ViewerConfigBuilderTest {

	@Test		@Test
	fun processClass_nonUnique() {		fun processClass_nonUnique() {
	Mockito.`when`(processor.process(any(CompilationUnit::class.java),		configBuilder.addLogCalls(listOf(
	any(ProtoLogCallVisitor::class.java), any(String::class.java)))		LogCall(TEST1.messageString, LogLevel.INFO, GROUP1, PATH),
	.thenAnswer { invocation ->		LogCall(TEST1.messageString, LogLevel.INFO, GROUP1, PATH),
	val visitor = invocation.arguments[1] as ProtoLogCallVisitor		LogCall(TEST1.messageString, LogLevel.INFO, GROUP1, PATH)).withContext())

	visitor.processCall(MethodCallExpr(), TEST1.messageString, LogLevel.INFO,
	GROUP1)
	visitor.processCall(MethodCallExpr(), TEST1.messageString, LogLevel.INFO,
	GROUP1)
	visitor.processCall(MethodCallExpr(), TEST1.messageString, LogLevel.INFO,
	GROUP1)

	invocation.arguments[0] as CompilationUnit
	}

	configBuilder.processClass(dummyCompilationUnit, PATH)

	val parsedConfig = parseConfig(configBuilder.build())		val parsedConfig = parseConfig(configBuilder.build())
	assertEquals(1, parsedConfig.size)		assertEquals(1, parsedConfig.size)
	@@ -103,28 +76,19 @@ class ViewerConfigBuilderTest {

	@Test		@Test
	fun processClass_disabled() {		fun processClass_disabled() {
	Mockito.`when`(processor.process(any(CompilationUnit::class.java),		configBuilder.addLogCalls(listOf(
	any(ProtoLogCallVisitor::class.java), any(String::class.java)))		LogCall(TEST1.messageString, LogLevel.INFO, GROUP1, PATH),
	.thenAnswer { invocation ->		LogCall(TEST2.messageString, LogLevel.DEBUG, GROUP_DISABLED, PATH),
	val visitor = invocation.arguments[1] as ProtoLogCallVisitor		LogCall(TEST3.messageString, LogLevel.ERROR, GROUP_TEXT_DISABLED, PATH))
			.withContext())
	visitor.processCall(MethodCallExpr(), TEST1.messageString, LogLevel.INFO,
	GROUP1)
	visitor.processCall(MethodCallExpr(), TEST2.messageString, LogLevel.DEBUG,
	LogGroup("DEBUG_GROUP", false, true, TAG2))
	visitor.processCall(MethodCallExpr(), TEST3.messageString, LogLevel.ERROR,
	LogGroup("DEBUG_GROUP", true, false, TAG2))

	invocation.arguments[0] as CompilationUnit
	}

	configBuilder.processClass(dummyCompilationUnit, PATH)

	val parsedConfig = parseConfig(configBuilder.build())		val parsedConfig = parseConfig(configBuilder.build())
	assertEquals(2, parsedConfig.size)		assertEquals(2, parsedConfig.size)
	assertEquals(TEST1, parsedConfig[CodeUtils.hash(PATH, TEST1.messageString,		assertEquals(TEST1, parsedConfig[CodeUtils.hash(
	LogLevel.INFO, GROUP1)])		PATH, TEST1.messageString, LogLevel.INFO, GROUP1)])
	assertEquals(TEST3, parsedConfig[CodeUtils.hash(PATH, TEST3.messageString,		assertEquals(TEST3, parsedConfig[CodeUtils.hash(
	LogLevel.ERROR, LogGroup("DEBUG_GROUP", true, false, TAG2))])		PATH, TEST3.messageString, LogLevel.ERROR, GROUP_TEXT_DISABLED)])
	}		}

			private fun List<LogCall>.withContext() = map { it to ParsingContext() }
	}		}