Merge "protolog: improve build time of frameworks/base/services" (e301ccf8) · Commits · e / os / android_frameworks_base

tools/protologtool/src/com/android/protolog/tool/ProtoLogTool.kt

+65 −31

Original line number	Diff line number	Diff line
		@@ -18,11 +18,14 @@ package com.android.protolog.tool

		import com.android.protolog.tool.CommandOptions.Companion.USAGE
		import com.github.javaparser.ParseProblemException
		import com.github.javaparser.ParserConfiguration
		import com.github.javaparser.StaticJavaParser
		import com.github.javaparser.ast.CompilationUnit
		import java.io.File
		import java.io.FileInputStream
		import java.io.FileOutputStream
		import java.util.concurrent.ExecutorService
		import java.util.concurrent.Executors
		import java.util.jar.JarOutputStream
		import java.util.zip.ZipEntry
		import kotlin.system.exitProcess
		@@ -45,16 +48,18 @@ object ProtoLogTool {
		val outJar = JarOutputStream(out)
		val processor = ProtoLogCallProcessor(command.protoLogClassNameArg,
		command.protoLogGroupsClassNameArg, groups)
		val transformer = SourceTransformer(command.protoLogImplClassNameArg, processor)

		command.javaSourceArgs.forEach { path ->
		val executor = newThreadPool()

		command.javaSourceArgs.map { path ->
		executor.submitCallable {
		val transformer = SourceTransformer(command.protoLogImplClassNameArg, processor)
		val file = File(path)
		val text = file.readText()
		val newPath = path
		val outSrc = try {
		val code = tryParse(text, path)
		if (containsProtoLogText(text, command.protoLogClassNameArg)) {
		transformer.processClass(text, newPath, code)
		transformer.processClass(text, path, code)
		} else {
		text
		}
		@@ -64,11 +69,17 @@ object ProtoLogTool {
		println("\n${ex.message}\n")
		text
		}
		outJar.putNextEntry(ZipEntry(newPath))
		path to outSrc
		}
		}.map { future ->
		val (path, outSrc) = future.get()
		outJar.putNextEntry(ZipEntry(path))
		outJar.write(outSrc.toByteArray())
		outJar.closeEntry()
		}

		executor.shutdown()

		outJar.close()
		out.close()
		}
		@@ -92,7 +103,11 @@ object ProtoLogTool {
		val processor = ProtoLogCallProcessor(command.protoLogClassNameArg,
		command.protoLogGroupsClassNameArg, groups)
		val builder = ViewerConfigBuilder(processor)
		command.javaSourceArgs.forEach { path ->

		val executor = newThreadPool()

		command.javaSourceArgs.map { path ->
		executor.submitCallable {
		val file = File(path)
		val text = file.readText()
		if (containsProtoLogText(text, command.protoLogClassNameArg)) {
		@@ -101,14 +116,23 @@ object ProtoLogTool {
		val pack = if (code.packageDeclaration.isPresent) code.packageDeclaration
		.get().nameAsString else ""
		val newPath = pack.replace('.', '/') + '/' + file.name
		builder.processClass(code, newPath)
		builder.findLogCalls(code, newPath)
		} catch (ex: ParsingException) {
		// If we cannot parse this file, skip it (and log why). Compilation will fail
		// in a subsequent build step.
		println("\n${ex.message}\n")
		null
		}
		} else {
		null
		}
		}
		}.forEach { future ->
		builder.addLogCalls(future.get() ?: return@forEach)
		}

		executor.shutdown()

		val out = FileOutputStream(command.viewerConfigJsonArg)
		out.write(builder.build().toByteArray())
		out.close()
		@@ -122,6 +146,11 @@ object ProtoLogTool {

		@JvmStatic
		fun main(args: Array<String>) {
		StaticJavaParser.setConfiguration(ParserConfiguration().apply {
		setLanguageLevel(ParserConfiguration.LanguageLevel.RAW)
		setAttributeComments(false)
		})

		try {
		val command = CommandOptions(args)
		when (command.command) {
		@@ -138,3 +167,8 @@ object ProtoLogTool {
		}
		}
		}

		private fun <T> ExecutorService.submitCallable(f: () -> T) = submit(f)

		private fun newThreadPool() = Executors.newFixedThreadPool(
		Runtime.getRuntime().availableProcessors())

tools/protologtool/src/com/android/protolog/tool/SourceTransformer.kt

+2 −4

Original line number	Diff line number	Diff line
		@@ -42,7 +42,6 @@ import com.github.javaparser.ast.type.PrimitiveType
		import com.github.javaparser.ast.type.Type
		import com.github.javaparser.printer.PrettyPrinter
		import com.github.javaparser.printer.PrettyPrinterConfiguration
		import com.github.javaparser.printer.lexicalpreservation.LexicalPreservingPrinter

		class SourceTransformer(
		protoLogImplClassName: String,
		@@ -135,7 +134,8 @@ class SourceTransformer(
		// Inline the new statement.
		val printedIfStmt = inlinePrinter.print(ifStmt)
		// Append blank lines to preserve line numbering in file (to allow debugging)
		val newLines = LexicalPreservingPrinter.print(parentStmt).count { c -> c == '\n' }
		val parentRange = parentStmt.range.get()
		val newLines = parentRange.end.line - parentRange.begin.line
		val newStmt = printedIfStmt.substringBeforeLast('}') + ("\n".repeat(newLines)) + '}'
		// pre-workaround code, see explanation below
		/*
		@@ -224,9 +224,7 @@ class SourceTransformer(
		fileName = path
		processedCode = code.split('\n').toMutableList()
		offsets = IntArray(processedCode.size)
		LexicalPreservingPrinter.setup(compilationUnit)
		protoLogCallProcessor.process(compilationUnit, this, fileName)
		// return LexicalPreservingPrinter.print(compilationUnit)
		return processedCode.joinToString("\n")
		}
		}

tools/protologtool/src/com/android/protolog/tool/ViewerConfigBuilder.kt

+35 −20

Original line number	Diff line number	Diff line
		@@ -23,39 +23,52 @@ import com.github.javaparser.ast.expr.MethodCallExpr
		import java.io.StringWriter

		class ViewerConfigBuilder(
		private val protoLogCallVisitor: ProtoLogCallProcessor
		) : ProtoLogCallVisitor {
		override fun processCall(
		call: MethodCallExpr,
		messageString: String,
		level: LogLevel,
		group: LogGroup
		private val processor: ProtoLogCallProcessor
		) {
		private fun addLogCall(logCall: LogCall, context: ParsingContext) {
		val group = logCall.logGroup
		val messageString = logCall.messageString
		if (group.enabled) {
		val position = fileName
		val key = CodeUtils.hash(position, messageString, level, group)
		val key = logCall.key()
		if (statements.containsKey(key)) {
		if (statements[key] != LogCall(messageString, level, group, position)) {
		if (statements[key] != logCall) {
		throw HashCollisionException(
		"Please modify the log message \"$messageString\" " +
		"or \"${statements[key]}\" - their hashes are equal.",
		ParsingContext(fileName, call))
		"or \"${statements[key]}\" - their hashes are equal.", context)
		}
		} else {
		groups.add(group)
		statements[key] = LogCall(messageString, level, group, position)
		call.range.isPresent
		statements[key] = logCall
		}
		}
		}

		private val statements: MutableMap<Int, LogCall> = mutableMapOf()
		private val groups: MutableSet<LogGroup> = mutableSetOf()
		private var fileName: String = ""

		fun processClass(unit: CompilationUnit, fileName: String) {
		this.fileName = fileName
		protoLogCallVisitor.process(unit, this, fileName)
		fun findLogCalls(unit: CompilationUnit, fileName: String): List<Pair<LogCall, ParsingContext>> {
		val calls = mutableListOf<Pair<LogCall, ParsingContext>>()
		val visitor = object : ProtoLogCallVisitor {
		override fun processCall(
		call: MethodCallExpr,
		messageString: String,
		level: LogLevel,
		group: LogGroup
		) {
		val logCall = LogCall(messageString, level, group, fileName)
		val context = ParsingContext(fileName, call)
		calls.add(logCall to context)
		}
		}
		processor.process(unit, visitor, fileName)

		return calls
		}

		fun addLogCalls(calls: List<Pair<LogCall, ParsingContext>>) {
		calls.forEach { (logCall, context) ->
		addLogCall(logCall, context)
		}
		}

		fun build(): String {
		@@ -101,5 +114,7 @@ class ViewerConfigBuilder(
		val logLevel: LogLevel,
		val logGroup: LogGroup,
		val position: String
		)
		) {
		fun key() = CodeUtils.hash(position, messageString, logLevel, logGroup)
		}
		}

tools/protologtool/tests/com/android/protolog/tool/ViewerConfigBuilderTest.kt

+23 −59

Original line number	Diff line number	Diff line
		@@ -17,8 +17,7 @@
		package com.android.protolog.tool

		import com.android.json.stream.JsonReader
		import com.github.javaparser.ast.CompilationUnit
		import com.github.javaparser.ast.expr.MethodCallExpr
		import com.android.protolog.tool.ViewerConfigBuilder.LogCall
		import org.junit.Assert.assertEquals
		import org.junit.Test
		import org.mockito.Mockito
		@@ -34,14 +33,12 @@ class ViewerConfigBuilderTest {
		private val GROUP1 = LogGroup("TEST_GROUP", true, true, TAG1)
		private val GROUP2 = LogGroup("DEBUG_GROUP", true, true, TAG2)
		private val GROUP3 = LogGroup("DEBUG_GROUP", true, true, TAG2)
		private val GROUP_DISABLED = LogGroup("DEBUG_GROUP", false, true, TAG2)
		private val GROUP_TEXT_DISABLED = LogGroup("DEBUG_GROUP", true, false, TAG2)
		private const val PATH = "/tmp/test.java"
		}

		private val processor: ProtoLogCallProcessor = Mockito.mock(ProtoLogCallProcessor::class.java)
		private val configBuilder = ViewerConfigBuilder(processor)
		private val dummyCompilationUnit = CompilationUnit()

		private fun <T> any(type: Class<T>): T = Mockito.any<T>(type)
		private val configBuilder = ViewerConfigBuilder(Mockito.mock(ProtoLogCallProcessor::class.java))

		private fun parseConfig(json: String): Map<Int, ViewerConfigParser.ConfigEntry> {
		return ViewerConfigParser().parseConfig(JsonReader(StringReader(json)))
		@@ -49,22 +46,10 @@ class ViewerConfigBuilderTest {

		@Test
		fun processClass() {
		Mockito.`when`(processor.process(any(CompilationUnit::class.java),
		any(ProtoLogCallVisitor::class.java), any(String::class.java)))
		.thenAnswer { invocation ->
		val visitor = invocation.arguments[1] as ProtoLogCallVisitor

		visitor.processCall(MethodCallExpr(), TEST1.messageString, LogLevel.INFO,
		GROUP1)
		visitor.processCall(MethodCallExpr(), TEST2.messageString, LogLevel.DEBUG,
		GROUP2)
		visitor.processCall(MethodCallExpr(), TEST3.messageString, LogLevel.ERROR,
		GROUP3)

		invocation.arguments[0] as CompilationUnit
		}

		configBuilder.processClass(dummyCompilationUnit, PATH)
		configBuilder.addLogCalls(listOf(
		LogCall(TEST1.messageString, LogLevel.INFO, GROUP1, PATH),
		LogCall(TEST2.messageString, LogLevel.DEBUG, GROUP2, PATH),
		LogCall(TEST3.messageString, LogLevel.ERROR, GROUP3, PATH)).withContext())

		val parsedConfig = parseConfig(configBuilder.build())
		assertEquals(3, parsedConfig.size)
		@@ -78,22 +63,10 @@ class ViewerConfigBuilderTest {

		@Test
		fun processClass_nonUnique() {
		Mockito.`when`(processor.process(any(CompilationUnit::class.java),
		any(ProtoLogCallVisitor::class.java), any(String::class.java)))
		.thenAnswer { invocation ->
		val visitor = invocation.arguments[1] as ProtoLogCallVisitor

		visitor.processCall(MethodCallExpr(), TEST1.messageString, LogLevel.INFO,
		GROUP1)
		visitor.processCall(MethodCallExpr(), TEST1.messageString, LogLevel.INFO,
		GROUP1)
		visitor.processCall(MethodCallExpr(), TEST1.messageString, LogLevel.INFO,
		GROUP1)

		invocation.arguments[0] as CompilationUnit
		}

		configBuilder.processClass(dummyCompilationUnit, PATH)
		configBuilder.addLogCalls(listOf(
		LogCall(TEST1.messageString, LogLevel.INFO, GROUP1, PATH),
		LogCall(TEST1.messageString, LogLevel.INFO, GROUP1, PATH),
		LogCall(TEST1.messageString, LogLevel.INFO, GROUP1, PATH)).withContext())

		val parsedConfig = parseConfig(configBuilder.build())
		assertEquals(1, parsedConfig.size)
		@@ -103,28 +76,19 @@ class ViewerConfigBuilderTest {

		@Test
		fun processClass_disabled() {
		Mockito.`when`(processor.process(any(CompilationUnit::class.java),
		any(ProtoLogCallVisitor::class.java), any(String::class.java)))
		.thenAnswer { invocation ->
		val visitor = invocation.arguments[1] as ProtoLogCallVisitor

		visitor.processCall(MethodCallExpr(), TEST1.messageString, LogLevel.INFO,
		GROUP1)
		visitor.processCall(MethodCallExpr(), TEST2.messageString, LogLevel.DEBUG,
		LogGroup("DEBUG_GROUP", false, true, TAG2))
		visitor.processCall(MethodCallExpr(), TEST3.messageString, LogLevel.ERROR,
		LogGroup("DEBUG_GROUP", true, false, TAG2))

		invocation.arguments[0] as CompilationUnit
		}

		configBuilder.processClass(dummyCompilationUnit, PATH)
		configBuilder.addLogCalls(listOf(
		LogCall(TEST1.messageString, LogLevel.INFO, GROUP1, PATH),
		LogCall(TEST2.messageString, LogLevel.DEBUG, GROUP_DISABLED, PATH),
		LogCall(TEST3.messageString, LogLevel.ERROR, GROUP_TEXT_DISABLED, PATH))
		.withContext())

		val parsedConfig = parseConfig(configBuilder.build())
		assertEquals(2, parsedConfig.size)
		assertEquals(TEST1, parsedConfig[CodeUtils.hash(PATH, TEST1.messageString,
		LogLevel.INFO, GROUP1)])
		assertEquals(TEST3, parsedConfig[CodeUtils.hash(PATH, TEST3.messageString,
		LogLevel.ERROR, LogGroup("DEBUG_GROUP", true, false, TAG2))])
		assertEquals(TEST1, parsedConfig[CodeUtils.hash(
		PATH, TEST1.messageString, LogLevel.INFO, GROUP1)])
		assertEquals(TEST3, parsedConfig[CodeUtils.hash(
		PATH, TEST3.messageString, LogLevel.ERROR, GROUP_TEXT_DISABLED)])
		}

		private fun List<LogCall>.withContext() = map { it to ParsingContext() }
		}