startop: add compiler type support to host and device switch. (75015c32) · Commits · e / os / android_frameworks_base

startop/scripts/app_startup/app_startup_runner.py

+30 −18

Original line number	Diff line number	Diff line
		@@ -41,11 +41,12 @@ DIR = os.path.abspath(os.path.dirname(__file__))
		sys.path.append(os.path.dirname(DIR))
		import lib.cmd_utils as cmd_utils
		import lib.print_utils as print_utils
		import iorap.compiler as compiler
		from app_startup.run_app_with_prefetch import PrefetchAppRunner
		import app_startup.lib.args_utils as args_utils
		from app_startup.lib.data_frame import DataFrame
		from app_startup.lib.perfetto_trace_collector import PerfettoTraceCollector
		from iorap.compiler import CompilerType
		import iorap.compiler as compiler

		# The following command line options participate in the combinatorial generation.
		# All other arguments have a global effect.
		@@ -58,8 +59,6 @@ _RUN_SCRIPT = os.path.join(os.path.dirname(os.path.realpath(__file__)),

		CollectorPackageInfo = NamedTuple('CollectorPackageInfo',
		[('package', str), ('compiler_filter', str)])
		_COMPILER_SCRIPT = os.path.join(os.path.dirname(os.path.dirname(
		os.path.realpath(__file__))), 'iorap/compiler.py')
		# by 2; systrace starts up slowly.

		_UNLOCK_SCREEN_SCRIPT = os.path.join(
		@@ -135,6 +134,10 @@ def parse_options(argv: List[str] = None):
		action='append',
		help='The trace duration (milliseconds) in '
		'compilation')
		optional_named.add_argument('--compiler-type', dest='compiler_type',
		type=CompilerType, choices=list(CompilerType),
		default=CompilerType.DEVICE,
		help='The type of compiler.')

		return parser.parse_args(argv)

		@@ -211,26 +214,26 @@ def parse_run_script_csv_file(csv_file: TextIO) -> DataFrame:

		return DataFrame(d)

		def compile_perfetto_trace(inodes_path: str,
		def build_ri_compiler_argv(inodes_path: str,
		perfetto_trace_file: str,
		trace_duration: Optional[timedelta]) -> TextIO:
		compiler_trace_file = tempfile.NamedTemporaryFile()
		argv = [_COMPILER_SCRIPT, '-i', inodes_path, '--perfetto-trace',
		perfetto_trace_file, '-o', compiler_trace_file.name]
		trace_duration: Optional[timedelta]
		) -> str:
		argv = ['-i', inodes_path, '--perfetto-trace',
		perfetto_trace_file]

		if trace_duration is not None:
		argv += ['--duration', str(int(trace_duration.total_seconds()
		* PerfettoTraceCollector.MS_PER_SEC))]

		print_utils.debug_print(argv)
		compiler.main(argv)
		return compiler_trace_file
		return argv

		def execute_run_using_perfetto_trace(collector_info,
		run_combos: Iterable[RunCommandArgs],
		simulate: bool,
		inodes_path: str,
		timeout: int) -> DataFrame:
		timeout: int,
		compiler_type: CompilerType) -> DataFrame:
		""" Executes run based on perfetto trace. """
		passed, perfetto_trace_file = run_perfetto_collector(collector_info,
		timeout,
		@@ -244,9 +247,15 @@ def execute_run_using_perfetto_trace(collector_info,
		if simulate:
		compiler_trace_file = tempfile.NamedTemporaryFile()
		else:
		compiler_trace_file = compile_perfetto_trace(inodes_path,
		ri_compiler_argv = build_ri_compiler_argv(inodes_path,
		perfetto_trace_file.name,
		combos.trace_duration)
		compiler_trace_file = compiler.compile(compiler_type,
		inodes_path,
		ri_compiler_argv,
		combos.package,
		combos.activity)

		with compiler_trace_file:
		combos = combos._replace(input=compiler_trace_file.name)
		print_utils.debug_print(combos)
		@@ -261,7 +270,8 @@ def execute_run_combos(
		grouped_run_combos: Iterable[Tuple[CollectorPackageInfo, Iterable[RunCommandArgs]]],
		simulate: bool,
		inodes_path: str,
		timeout: int):
		timeout: int,
		compiler_type: CompilerType):
		# nothing will work if the screen isn't unlocked first.
		cmd_utils.execute_arbitrary_command([_UNLOCK_SCREEN_SCRIPT],
		timeout,
		@@ -273,7 +283,8 @@ def execute_run_combos(
		run_combos,
		simulate,
		inodes_path,
		timeout)
		timeout,
		compiler_type)

		def gather_results(commands: Iterable[Tuple[DataFrame]],
		key_list: List[str], value_list: List[Tuple[str, ...]]):
		@@ -361,7 +372,8 @@ def main():
		exec = execute_run_combos(grouped_combos(),
		opts.simulate,
		opts.inodes,
		opts.timeout)
		opts.timeout,
		opts.compiler_type)

		results = gather_results(exec, _COMBINATORIAL_OPTIONS, combos())

startop/scripts/app_startup/app_startup_runner_test.py

+1 −1

Original line number	Diff line number	Diff line
		@@ -92,7 +92,7 @@ def default_dict_for_parsed_args(**kwargs):
		"""
		d = {'compiler_filters': None, 'simulate': False, 'debug': False,
		'output': None, 'timeout': 10, 'loop_count': 1, 'inodes': None,
		'trace_duration': None}
		'trace_duration': None, 'compiler_type': asr.CompilerType.HOST}
		d.update(kwargs)
		return d

startop/scripts/iorap/compiler.py

100755 → 100644

+53 −303

Original line number	Diff line number	Diff line
		#!/usr/bin/env python3

		#
		# Copyright (C) 2019 The Android Open Source Project
		# Copyright 2019, The Android Open Source Project
		#
		# Licensed under the Apache License, Version 2.0 (the "License");
		# you may not use this file except in compliance with the License.
		@@ -14,310 +13,61 @@
		# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
		# See the License for the specific language governing permissions and
		# limitations under the License.
		#

		#
		# Dependencies:
		#
		# $> sudo apt-get install python3-pip
		# $> pip3 install --user protobuf sqlalchemy sqlite3
		#

		import optparse
		import importlib
		import os
		import re
		import sys
		import tempfile
		from pathlib import Path
		from datetime import timedelta
		from typing import Iterable, Optional, List
		from enum import Enum
		from typing import TextIO, List

		# local import
		DIR = os.path.abspath(os.path.dirname(__file__))
		sys.path.append(os.path.dirname(DIR))
		from iorap.generated.TraceFile_pb2 import *
		from iorap.lib.inode2filename import Inode2Filename

		parent_dir_name = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
		sys.path.append(parent_dir_name)
		from trace_analyzer.lib.trace2db import Trace2Db, MmFilemapAddToPageCache, \
		RawFtraceEntry
		import lib.cmd_utils as cmd_utils

		_PAGE_SIZE = 4096 # adb shell getconf PAGESIZE ## size of a memory page in bytes.
		ANDROID_BUILD_TOP = Path(parent_dir_name).parents[3]
		TRACECONV_BIN = ANDROID_BUILD_TOP.joinpath(
		'external/perfetto/tools/traceconv')

		class PageRun:
		"""
		Intermediate representation for a run of one or more pages.
		"""
		def __init__(self, device_number: int, inode: int, offset: int, length: int):
		self.device_number = device_number
		self.inode = inode
		self.offset = offset
		self.length = length

		def __str__(self):
		return "PageRun(device_number=%d, inode=%d, offset=%d, length=%d)" \
		%(self.device_number, self.inode, self.offset, self.length)

		def debug_print(msg):
		#print(msg)
		pass

		UNDER_LAUNCH = False

		def page_cache_entries_to_runs(page_cache_entries: Iterable[MmFilemapAddToPageCache]):
		global _PAGE_SIZE

		runs = [
		PageRun(device_number=pg_entry.dev, inode=pg_entry.ino, offset=pg_entry.ofs,
		length=_PAGE_SIZE)
		for pg_entry in page_cache_entries
		]

		for r in runs:
		debug_print(r)

		print("Stats: Page runs totaling byte length: %d" %(len(runs) * _PAGE_SIZE))

		return runs

		def optimize_page_runs(page_runs):
		new_entries = []
		last_entry = None
		for pg_entry in page_runs:
		if last_entry:
		if pg_entry.device_number == last_entry.device_number and pg_entry.inode == last_entry.inode:
		# we are dealing with a run for the same exact file as a previous run.
		if pg_entry.offset == last_entry.offset + last_entry.length:
		# trivially contiguous entries. merge them together.
		last_entry.length += pg_entry.length
		continue
		# Default: Add the run without merging it to a previous run.
		last_entry = pg_entry
		new_entries.append(pg_entry)
		return new_entries

		def is_filename_matching_filter(file_name, filters=[]):
		"""
		Blacklist-style regular expression filters.

		:return: True iff file_name has an RE match in one of the filters.
		"""
		for filt in filters:
		res = re.search(filt, file_name)
		if res:
		return True

		return False

		def build_protobuf(page_runs, inode2filename, filters=[]):
		trace_file = TraceFile()
		trace_file_index = trace_file.index

		file_id_counter = 0
		file_id_map = {} # filename -> id

		stats_length_total = 0
		filename_stats = {} # filename -> total size

		skipped_inode_map = {}
		filtered_entry_map = {} # filename -> count

		for pg_entry in page_runs:
		fn = inode2filename.resolve(pg_entry.device_number, pg_entry.inode)
		if not fn:
		skipped_inode_map[pg_entry.inode] = skipped_inode_map.get(pg_entry.inode, 0) + 1
		continue

		filename = fn

		if filters and not is_filename_matching_filter(filename, filters):
		filtered_entry_map[filename] = filtered_entry_map.get(filename, 0) + 1
		continue

		file_id = file_id_map.get(filename)
		if not file_id:
		file_id = file_id_counter
		file_id_map[filename] = file_id_counter
		file_id_counter = file_id_counter + 1

		file_index_entry = trace_file_index.entries.add()
		file_index_entry.id = file_id
		file_index_entry.file_name = filename

		# already in the file index, add the file entry.
		file_entry = trace_file.list.entries.add()
		file_entry.index_id = file_id
		file_entry.file_length = pg_entry.length
		stats_length_total += file_entry.file_length
		file_entry.file_offset = pg_entry.offset

		filename_stats[filename] = filename_stats.get(filename, 0) + file_entry.file_length

		for inode, count in skipped_inode_map.items():
		print("WARNING: Skip inode %s because it's not in inode map (%d entries)" %(inode, count))

		print("Stats: Sum of lengths %d" %(stats_length_total))

		if filters:
		print("Filter: %d total files removed." %(len(filtered_entry_map)))

		for fn, count in filtered_entry_map.items():
		print("Filter: File '%s' removed '%d' entries." %(fn, count))

		for filename, file_size in filename_stats.items():
		print("%s,%s" %(filename, file_size))

		return trace_file

		def calc_trace_end_time(trace2db: Trace2Db,
		trace_duration: Optional[timedelta]) -> float:
		"""
		Calculates the end time based on the trace duration.
		The start time is the first receiving mm file map event.
		The end time is the start time plus the trace duration.
		All of them are in milliseconds.
		"""
		# If the duration is not set, assume all time is acceptable.
		if trace_duration is None:
		# float('inf')
		return RawFtraceEntry.__table__.c.timestamp.type.python_type('inf')

		first_event = trace2db.session.query(MmFilemapAddToPageCache).join(
		MmFilemapAddToPageCache.raw_ftrace_entry).order_by(
		RawFtraceEntry.timestamp).first()

		# total_seconds() will return a float number.
		return first_event.raw_ftrace_entry.timestamp + trace_duration.total_seconds()

		def query_add_to_page_cache(trace2db: Trace2Db, trace_duration: Optional[timedelta]):
		end_time = calc_trace_end_time(trace2db, trace_duration)
		# SELECT * FROM tbl ORDER BY id;
		return trace2db.session.query(MmFilemapAddToPageCache).join(
		MmFilemapAddToPageCache.raw_ftrace_entry).filter(
		RawFtraceEntry.timestamp <= end_time).order_by(
		MmFilemapAddToPageCache.id).all()

		def transform_perfetto_trace_to_systrace(path_to_perfetto_trace: str,
		path_to_tmp_systrace: str) -> None:
		""" Transforms the systrace file from perfetto trace. """
		cmd_utils.run_command_nofail([str(TRACECONV_BIN),
		'systrace',
		path_to_perfetto_trace,
		path_to_tmp_systrace])


		def run(sql_db_path:str,
		trace_file:str,
		trace_duration:Optional[timedelta],
		output_file:str,
		inode_table:str,
		filter:List[str]) -> int:
		trace2db = Trace2Db(sql_db_path)
		# Speed optimization: Skip any entries that aren't mm_filemap_add_to_pagecache.
		trace2db.set_raw_ftrace_entry_filter(\
		lambda entry: entry['function'] == 'mm_filemap_add_to_page_cache')
		# TODO: parse multiple trace files here.
		parse_count = trace2db.parse_file_into_db(trace_file)

		mm_filemap_add_to_page_cache_rows = query_add_to_page_cache(trace2db,
		trace_duration)
		print("DONE. Parsed %d entries into sql db." %(len(mm_filemap_add_to_page_cache_rows)))

		page_runs = page_cache_entries_to_runs(mm_filemap_add_to_page_cache_rows)
		print("DONE. Converted %d entries" %(len(page_runs)))

		# TODO: flags to select optimizations.
		optimized_page_runs = optimize_page_runs(page_runs)
		print("DONE. Optimized down to %d entries" %(len(optimized_page_runs)))

		print("Build protobuf...")
		trace_file = build_protobuf(optimized_page_runs, inode_table, filter)

		print("Write protobuf to file...")
		output_file = open(output_file, 'wb')
		output_file.write(trace_file.SerializeToString())
		output_file.close()

		print("DONE")

		# TODO: Silent running mode [no output except on error] for build runs.

		return 0

		def main(argv):
		parser = optparse.OptionParser(usage="Usage: %prog [options]", description="Compile systrace file into TraceFile.pb")
		parser.add_option('-i', dest='inode_data_file', metavar='FILE',
		help='Read cached inode data from a file saved earlier with pagecache.py -d')
		parser.add_option('-t', dest='trace_file', metavar='FILE',
		help='Path to systrace file (trace.html) that will be parsed')
		parser.add_option('--perfetto-trace', dest='perfetto_trace_file',
		metavar='FILE',
		help='Path to perfetto trace that will be parsed')

		parser.add_option('--db', dest='sql_db', metavar='FILE',
		help='Path to intermediate sqlite3 database [default: in-memory].')

		parser.add_option('-f', dest='filter', action="append", default=[],
		help="Add file filter. All file entries not matching one of the filters are discarded.")

		parser.add_option('-l', dest='launch_lock', action="store_true", default=False,
		help="Exclude all events not inside launch_lock")

		parser.add_option('-o', dest='output_file', metavar='FILE',
		help='Output protobuf file')

		parser.add_option('--duration', dest='trace_duration', action="store",
		type=int, help='The duration of trace in milliseconds.')

		options, categories = parser.parse_args(argv[1:])

		# TODO: OptionParser should have some flags to make these mandatory.
		if not options.inode_data_file:
		parser.error("-i is required")
		if not options.trace_file and not options.perfetto_trace_file:
		parser.error("one of -t or --perfetto-trace is required")
		if options.trace_file and options.perfetto_trace_file:
		parser.error("please enter either -t or --perfetto-trace, not both")
		if not options.output_file:
		parser.error("-o is required")

		if options.launch_lock:
		print("INFO: Launch lock flag (-l) enabled; filtering all events not inside launch_lock.")

		inode_table = Inode2Filename.new_from_filename(options.inode_data_file)

		sql_db_path = ":memory:"
		if options.sql_db:
		sql_db_path = options.sql_db

		trace_duration = timedelta(milliseconds=options.trace_duration) if \
		options.trace_duration is not None else None

		# if the input is systrace
		if options.trace_file:
		return run(sql_db_path,
		options.trace_file,
		trace_duration,
		options.output_file,
		inode_table,
		options.filter)

		# if the input is perfetto trace
		# TODO python 3.7 switch to using nullcontext
		with tempfile.NamedTemporaryFile() as trace_file:
		transform_perfetto_trace_to_systrace(options.perfetto_trace_file,
		trace_file.name)
		return run(sql_db_path,
		trace_file.name,
		trace_duration,
		options.output_file,
		inode_table,
		options.filter)

		if __name__ == '__main__':
		print(sys.argv)
		sys.exit(main(sys.argv))
		import lib.print_utils as print_utils

		# Type of compiler.
		class CompilerType(Enum):
		HOST = 1 # iorap.cmd.compiler on host
		DEVICE = 2 # adb shell iorap.cmd.compiler
		RI = 3 # compiler.py

		def compile_perfetto_trace_ri(
		argv: List[str],
		compiler) -> TextIO:
		print_utils.debug_print('Compile using RI compiler.')
		compiler_trace_file = tempfile.NamedTemporaryFile()
		argv.extend(['-o', compiler_trace_file.name])
		print_utils.debug_print(argv)
		compiler.main([''] + argv)
		return compiler_trace_file

		def compile_perfetto_trace_device(inodes_path: str,
		package: str,
		activity: str,
		compiler) -> TextIO:
		print_utils.debug_print('Compile using on-device compiler.')
		compiler_trace_file = tempfile.NamedTemporaryFile()
		compiler.main(inodes_path, package, activity, compiler_trace_file.name)
		return compiler_trace_file

		def compile(compiler_type: CompilerType,
		inodes_path: str,
		ri_compiler_argv,
		package: str,
		activity: str) -> TextIO:
		if compiler_type == CompilerType.RI:
		compiler = importlib.import_module('iorap.compiler_ri')
		compiler_trace_file = compile_perfetto_trace_ri(ri_compiler_argv,
		compiler)
		return compiler_trace_file
		if compiler_type == CompilerType.DEVICE:
		compiler = importlib.import_module('iorap.compiler_device')
		compiler_trace_file = compile_perfetto_trace_device(inodes_path,
		package,
		activity,
		compiler)
		return compiler_trace_file

		# Should not arrive here.
		raise ValueError('Unknown compiler type')

startop/scripts/iorap/compiler_device.py

0 → 100644

+68 −0

Original line number	Diff line number	Diff line
		#!/usr/bin/env python3
		#
		# Copyright 2019, The Android Open Source Project
		#
		# Licensed under the Apache License, Version 2.0 (the "License");
		# you may not use this file except in compliance with the License.
		# You may obtain a copy of the License at
		#
		# http://www.apache.org/licenses/LICENSE-2.0
		#
		# Unless required by applicable law or agreed to in writing, software
		# distributed under the License is distributed on an "AS IS" BASIS,
		# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
		# See the License for the specific language governing permissions and
		# limitations under the License.

		import argparse
		import os
		import sys
		from typing import List

		DIR = os.path.abspath(os.path.dirname(__file__))
		sys.path.append(os.path.dirname(DIR)) # framework/base/startop/script
		import lib.print_utils as print_utils
		import iorap.lib.iorapd_utils as iorapd_utils
		from app_startup.lib.app_runner import AppRunner

		IORAP_COMMON_BASH_SCRIPT = os.path.join(DIR, 'common')

		def parse_options(argv: List[str] = None):
		"""Parses command line arguments and returns an argparse Namespace object."""
		parser = argparse.ArgumentParser(description="Compile perfetto trace file")
		required_named = parser.add_argument_group('required named arguments')

		required_named.add_argument('-i', dest='inodes', metavar='FILE',
		help='Read cached inode data from a file saved '
		'earlier with pagecache.py -d')
		required_named.add_argument('-p', dest='package',
		help='Package of the app to be compiled')

		optional_named = parser.add_argument_group('optional named arguments')
		optional_named.add_argument('-o', dest='output',
		help='The compiled trace is stored into the output file')
		optional_named.add_argument('-a', dest='activity',
		help='Activity of the app to be compiled')
		optional_named.add_argument('-d', dest='debug', action='store_true'
		, help='Activity of the app to be compiled')

		return parser.parse_args(argv)

		def main(inodes, package, activity, output, **kwargs) -> int:
		"""Entries of the program."""
		if not activity:
		activity = AppRunner.get_activity(package)

		passed = iorapd_utils.compile_perfetto_trace_on_device(package, activity,
		inodes)
		if passed and output:
		iorapd_utils.get_iorapd_compiler_trace(package, activity, output)

		return 0

		if __name__ == '__main__':
		opts = parse_options()
		if opts.debug:
		print_utils.DEBUG = opts.debug
		print_utils.debug_print(opts)
		sys.exit(main(**(vars(opts))))

startop/scripts/iorap/compiler_ri.py

0 → 100755

+323 −0

File added.

Preview size limit exceeded, changes collapsed.