Loading startop/scripts/iorap/compiler.py 0 → 100755 +244 −0 Original line number Diff line number Diff line #!/usr/bin/env python3 # # Copyright (C) 2019 The Android Open Source Project # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # # Dependencies: # # $> sudo apt-get install python3-pip # $> pip3 install --user protobuf sqlalchemy sqlite3 # import collections import optparse import os import re import sys from typing import Iterable from lib.inode2filename import Inode2Filename from generated.TraceFile_pb2 import * parent_dir_name = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) sys.path.append(parent_dir_name + "/trace_analyzer") from lib.trace2db import Trace2Db, MmFilemapAddToPageCache _PAGE_SIZE = 4096 # adb shell getconf PAGESIZE ## size of a memory page in bytes. class PageRun: """ Intermediate representation for a run of one or more pages. """ def __init__(self, device_number: int, inode: int, offset: int, length: int): self.device_number = device_number self.inode = inode self.offset = offset self.length = length def __str__(self): return "PageRun(device_number=%d, inode=%d, offset=%d, length=%d)" \ %(self.device_number, self.inode, self.offset, self.length) def debug_print(msg): #print(msg) pass UNDER_LAUNCH = False def page_cache_entries_to_runs(page_cache_entries: Iterable[MmFilemapAddToPageCache]): global _PAGE_SIZE runs = [ PageRun(device_number=pg_entry.dev, inode=pg_entry.ino, offset=pg_entry.ofs, length=_PAGE_SIZE) for pg_entry in page_cache_entries ] for r in runs: debug_print(r) print("Stats: Page runs totaling byte length: %d" %(len(runs) * _PAGE_SIZE)) return runs def optimize_page_runs(page_runs): new_entries = [] last_entry = None for pg_entry in page_runs: if last_entry: if pg_entry.device_number == last_entry.device_number and pg_entry.inode == last_entry.inode: # we are dealing with a run for the same exact file as a previous run. if pg_entry.offset == last_entry.offset + last_entry.length: # trivially contiguous entries. merge them together. last_entry.length += pg_entry.length continue # Default: Add the run without merging it to a previous run. last_entry = pg_entry new_entries.append(pg_entry) return new_entries def is_filename_matching_filter(file_name, filters=[]): """ Blacklist-style regular expression filters. :return: True iff file_name has an RE match in one of the filters. """ for filt in filters: res = re.search(filt, file_name) if res: return True return False def build_protobuf(page_runs, inode2filename, filters=[]): trace_file = TraceFile() trace_file_index = trace_file.index file_id_counter = 0 file_id_map = {} # filename -> id stats_length_total = 0 filename_stats = {} # filename -> total size skipped_inode_map = {} filtered_entry_map = {} # filename -> count for pg_entry in page_runs: fn = inode2filename.resolve(pg_entry.device_number, pg_entry.inode) if not fn: skipped_inode_map[pg_entry.inode] = skipped_inode_map.get(pg_entry.inode, 0) + 1 continue filename = fn if filters and not is_filename_matching_filter(filename, filters): filtered_entry_map[filename] = filtered_entry_map.get(filename, 0) + 1 continue file_id = file_id_map.get(filename) if not file_id: file_id = file_id_counter file_id_map[filename] = file_id_counter file_id_counter = file_id_counter + 1 file_index_entry = trace_file_index.entries.add() file_index_entry.id = file_id file_index_entry.file_name = filename # already in the file index, add the file entry. file_entry = trace_file.list.entries.add() file_entry.index_id = file_id file_entry.file_length = pg_entry.length stats_length_total += file_entry.file_length file_entry.file_offset = pg_entry.offset filename_stats[filename] = filename_stats.get(filename, 0) + file_entry.file_length for inode, count in skipped_inode_map.items(): print("WARNING: Skip inode %s because it's not in inode map (%d entries)" %(inode, count)) print("Stats: Sum of lengths %d" %(stats_length_total)) if filters: print("Filter: %d total files removed." %(len(filtered_entry_map))) for fn, count in filtered_entry_map.items(): print("Filter: File '%s' removed '%d' entries." %(fn, count)) for filename, file_size in filename_stats.items(): print("%s,%s" %(filename, file_size)) return trace_file def query_add_to_page_cache(trace2db: Trace2Db): # SELECT * FROM tbl ORDER BY id; return trace2db.session.query(MmFilemapAddToPageCache).order_by(MmFilemapAddToPageCache.id).all() def main(argv): parser = optparse.OptionParser(usage="Usage: %prog [options]", description="Compile systrace file into TraceFile.pb") parser.add_option('-i', dest='inode_data_file', metavar='FILE', help='Read cached inode data from a file saved earlier with pagecache.py -d') parser.add_option('-t', dest='trace_file', metavar='FILE', help='Path to systrace file (trace.html) that will be parsed') parser.add_option('--db', dest='sql_db', metavar='FILE', help='Path to intermediate sqlite3 database [default: in-memory].') parser.add_option('-f', dest='filter', action="append", default=[], help="Add file filter. All file entries not matching one of the filters are discarded.") parser.add_option('-l', dest='launch_lock', action="store_true", default=False, help="Exclude all events not inside launch_lock") parser.add_option('-o', dest='output_file', metavar='FILE', help='Output protobuf file') options, categories = parser.parse_args(argv[1:]) # TODO: OptionParser should have some flags to make these mandatory. if not options.inode_data_file: parser.error("-i is required") if not options.trace_file: parser.error("-t is required") if not options.output_file: parser.error("-o is required") if options.launch_lock: print("INFO: Launch lock flag (-l) enabled; filtering all events not inside launch_lock.") inode_table = Inode2Filename.new_from_filename(options.inode_data_file) trace_file = open(options.trace_file) sql_db_path = ":memory:" if options.sql_db: sql_db_path = options.sql_db trace2db = Trace2Db(sql_db_path) # Speed optimization: Skip any entries that aren't mm_filemap_add_to_pagecache. trace2db.set_raw_ftrace_entry_filter(\ lambda entry: entry['function'] == 'mm_filemap_add_to_page_cache') # TODO: parse multiple trace files here. parse_count = trace2db.parse_file_into_db(options.trace_file) mm_filemap_add_to_page_cache_rows = query_add_to_page_cache(trace2db) print("DONE. Parsed %d entries into sql db." %(len(mm_filemap_add_to_page_cache_rows))) page_runs = page_cache_entries_to_runs(mm_filemap_add_to_page_cache_rows) print("DONE. Converted %d entries" %(len(page_runs))) # TODO: flags to select optimizations. optimized_page_runs = optimize_page_runs(page_runs) print("DONE. Optimized down to %d entries" %(len(optimized_page_runs))) print("Build protobuf...") trace_file = build_protobuf(optimized_page_runs, inode_table, options.filter) print("Write protobuf to file...") output_file = open(options.output_file, 'wb') output_file.write(trace_file.SerializeToString()) output_file.close() print("DONE") # TODO: Silent running mode [no output except on error] for build runs. return 0 sys.exit(main(sys.argv)) startop/scripts/iorap/generated/TraceFile_pb2.py 0 → 100644 +259 −0 Original line number Diff line number Diff line # Generated by the protocol buffer compiler. DO NOT EDIT! # source: TraceFile.proto import sys _b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1')) from google.protobuf import descriptor as _descriptor from google.protobuf import message as _message from google.protobuf import reflection as _reflection from google.protobuf import symbol_database as _symbol_database from google.protobuf import descriptor_pb2 # @@protoc_insertion_point(imports) _sym_db = _symbol_database.Default() DESCRIPTOR = _descriptor.FileDescriptor( name='TraceFile.proto', package='iorap.serialize.proto', syntax='proto2', serialized_pb=_b('\n\x0fTraceFile.proto\x12\x15iorap.serialize.proto\"u\n\tTraceFile\x12\x34\n\x05index\x18\x01 \x02(\x0b\x32%.iorap.serialize.proto.TraceFileIndex\x12\x32\n\x04list\x18\x02 \x02(\x0b\x32$.iorap.serialize.proto.TraceFileList\"M\n\x0eTraceFileIndex\x12;\n\x07\x65ntries\x18\x01 \x03(\x0b\x32*.iorap.serialize.proto.TraceFileIndexEntry\"4\n\x13TraceFileIndexEntry\x12\n\n\x02id\x18\x01 \x02(\x03\x12\x11\n\tfile_name\x18\x02 \x02(\t\"G\n\rTraceFileList\x12\x36\n\x07\x65ntries\x18\x01 \x03(\x0b\x32%.iorap.serialize.proto.TraceFileEntry\"L\n\x0eTraceFileEntry\x12\x10\n\x08index_id\x18\x01 \x02(\x03\x12\x13\n\x0b\x66ile_offset\x18\x02 \x02(\x03\x12\x13\n\x0b\x66ile_length\x18\x03 \x02(\x03\x42\x1c\n\x18\x63om.google.android.iorapH\x03') ) _sym_db.RegisterFileDescriptor(DESCRIPTOR) _TRACEFILE = _descriptor.Descriptor( name='TraceFile', full_name='iorap.serialize.proto.TraceFile', filename=None, file=DESCRIPTOR, containing_type=None, fields=[ _descriptor.FieldDescriptor( name='index', full_name='iorap.serialize.proto.TraceFile.index', index=0, number=1, type=11, cpp_type=10, label=2, has_default_value=False, default_value=None, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), _descriptor.FieldDescriptor( name='list', full_name='iorap.serialize.proto.TraceFile.list', index=1, number=2, type=11, cpp_type=10, label=2, has_default_value=False, default_value=None, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), ], extensions=[ ], nested_types=[], enum_types=[ ], options=None, is_extendable=False, syntax='proto2', extension_ranges=[], oneofs=[ ], serialized_start=42, serialized_end=159, ) _TRACEFILEINDEX = _descriptor.Descriptor( name='TraceFileIndex', full_name='iorap.serialize.proto.TraceFileIndex', filename=None, file=DESCRIPTOR, containing_type=None, fields=[ _descriptor.FieldDescriptor( name='entries', full_name='iorap.serialize.proto.TraceFileIndex.entries', index=0, number=1, type=11, cpp_type=10, label=3, has_default_value=False, default_value=[], message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), ], extensions=[ ], nested_types=[], enum_types=[ ], options=None, is_extendable=False, syntax='proto2', extension_ranges=[], oneofs=[ ], serialized_start=161, serialized_end=238, ) _TRACEFILEINDEXENTRY = _descriptor.Descriptor( name='TraceFileIndexEntry', full_name='iorap.serialize.proto.TraceFileIndexEntry', filename=None, file=DESCRIPTOR, containing_type=None, fields=[ _descriptor.FieldDescriptor( name='id', full_name='iorap.serialize.proto.TraceFileIndexEntry.id', index=0, number=1, type=3, cpp_type=2, label=2, has_default_value=False, default_value=0, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), _descriptor.FieldDescriptor( name='file_name', full_name='iorap.serialize.proto.TraceFileIndexEntry.file_name', index=1, number=2, type=9, cpp_type=9, label=2, has_default_value=False, default_value=_b("").decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), ], extensions=[ ], nested_types=[], enum_types=[ ], options=None, is_extendable=False, syntax='proto2', extension_ranges=[], oneofs=[ ], serialized_start=240, serialized_end=292, ) _TRACEFILELIST = _descriptor.Descriptor( name='TraceFileList', full_name='iorap.serialize.proto.TraceFileList', filename=None, file=DESCRIPTOR, containing_type=None, fields=[ _descriptor.FieldDescriptor( name='entries', full_name='iorap.serialize.proto.TraceFileList.entries', index=0, number=1, type=11, cpp_type=10, label=3, has_default_value=False, default_value=[], message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), ], extensions=[ ], nested_types=[], enum_types=[ ], options=None, is_extendable=False, syntax='proto2', extension_ranges=[], oneofs=[ ], serialized_start=294, serialized_end=365, ) _TRACEFILEENTRY = _descriptor.Descriptor( name='TraceFileEntry', full_name='iorap.serialize.proto.TraceFileEntry', filename=None, file=DESCRIPTOR, containing_type=None, fields=[ _descriptor.FieldDescriptor( name='index_id', full_name='iorap.serialize.proto.TraceFileEntry.index_id', index=0, number=1, type=3, cpp_type=2, label=2, has_default_value=False, default_value=0, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), _descriptor.FieldDescriptor( name='file_offset', full_name='iorap.serialize.proto.TraceFileEntry.file_offset', index=1, number=2, type=3, cpp_type=2, label=2, has_default_value=False, default_value=0, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), _descriptor.FieldDescriptor( name='file_length', full_name='iorap.serialize.proto.TraceFileEntry.file_length', index=2, number=3, type=3, cpp_type=2, label=2, has_default_value=False, default_value=0, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), ], extensions=[ ], nested_types=[], enum_types=[ ], options=None, is_extendable=False, syntax='proto2', extension_ranges=[], oneofs=[ ], serialized_start=367, serialized_end=443, ) _TRACEFILE.fields_by_name['index'].message_type = _TRACEFILEINDEX _TRACEFILE.fields_by_name['list'].message_type = _TRACEFILELIST _TRACEFILEINDEX.fields_by_name['entries'].message_type = _TRACEFILEINDEXENTRY _TRACEFILELIST.fields_by_name['entries'].message_type = _TRACEFILEENTRY DESCRIPTOR.message_types_by_name['TraceFile'] = _TRACEFILE DESCRIPTOR.message_types_by_name['TraceFileIndex'] = _TRACEFILEINDEX DESCRIPTOR.message_types_by_name['TraceFileIndexEntry'] = _TRACEFILEINDEXENTRY DESCRIPTOR.message_types_by_name['TraceFileList'] = _TRACEFILELIST DESCRIPTOR.message_types_by_name['TraceFileEntry'] = _TRACEFILEENTRY TraceFile = _reflection.GeneratedProtocolMessageType('TraceFile', (_message.Message,), dict( DESCRIPTOR = _TRACEFILE, __module__ = 'TraceFile_pb2' # @@protoc_insertion_point(class_scope:iorap.serialize.proto.TraceFile) )) _sym_db.RegisterMessage(TraceFile) TraceFileIndex = _reflection.GeneratedProtocolMessageType('TraceFileIndex', (_message.Message,), dict( DESCRIPTOR = _TRACEFILEINDEX, __module__ = 'TraceFile_pb2' # @@protoc_insertion_point(class_scope:iorap.serialize.proto.TraceFileIndex) )) _sym_db.RegisterMessage(TraceFileIndex) TraceFileIndexEntry = _reflection.GeneratedProtocolMessageType('TraceFileIndexEntry', (_message.Message,), dict( DESCRIPTOR = _TRACEFILEINDEXENTRY, __module__ = 'TraceFile_pb2' # @@protoc_insertion_point(class_scope:iorap.serialize.proto.TraceFileIndexEntry) )) _sym_db.RegisterMessage(TraceFileIndexEntry) TraceFileList = _reflection.GeneratedProtocolMessageType('TraceFileList', (_message.Message,), dict( DESCRIPTOR = _TRACEFILELIST, __module__ = 'TraceFile_pb2' # @@protoc_insertion_point(class_scope:iorap.serialize.proto.TraceFileList) )) _sym_db.RegisterMessage(TraceFileList) TraceFileEntry = _reflection.GeneratedProtocolMessageType('TraceFileEntry', (_message.Message,), dict( DESCRIPTOR = _TRACEFILEENTRY, __module__ = 'TraceFile_pb2' # @@protoc_insertion_point(class_scope:iorap.serialize.proto.TraceFileEntry) )) _sym_db.RegisterMessage(TraceFileEntry) DESCRIPTOR.has_options = True DESCRIPTOR._options = _descriptor._ParseOptions(descriptor_pb2.FileOptions(), _b('\n\030com.google.android.iorapH\003')) # @@protoc_insertion_point(module_scope) startop/scripts/iorap/generated/codegen_protos 0 → 100755 +35 −0 Original line number Diff line number Diff line #!/bin/bash # # Copyright 2019, The Android Open Source Project # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" APROTOC="$(which aprotoc)" IORAP_SERIALIZE_DIR="${DIR}/../../../../../../system/iorap/src/serialize" IORAP_PROTOS=($IORAP_SERIALIZE_DIR/*.proto) if [[ $? -ne 0 ]]; then echo "Fatal: Missing aprotoc. Set APROTOC=... or lunch build/envsetup.sh?" >&2 exit 1 fi if ! [[ -d $IORAP_SERIALIZE_DIR ]]; then echo "Fatal: Directory '$IORAP_SERIALIZE_DIR' does not exist." >&2 exit 1 fi # codegen the .py files into the same directory as this script. echo "$APROTOC" --proto_path="$IORAP_SERIALIZE_DIR" --python_out="$DIR" "${IORAP_PROTOS[@]}" "$APROTOC" --proto_path="$IORAP_SERIALIZE_DIR" --python_out="$DIR" "${IORAP_PROTOS[@]}" startop/scripts/iorap/lib/inode2filename.py 0 → 100644 +94 −0 Original line number Diff line number Diff line #!/usr/bin/env python3 # # Copyright (C) 2019 The Android Open Source Project # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # from typing import Any, Callable, Dict, Generic, Iterable, List, NamedTuple, TextIO, Tuple, TypeVar, Optional, Union, TextIO import re class Inode2Filename: """ Parses a text file of the format "uint(dev_t) uint(ino_t) int(file_size) string(filepath)\\n"* Lines not matching this format are ignored. """ def __init__(self, inode_data_file: TextIO): """ Create an Inode2Filename that reads cached inode from a file saved earlier (e.g. with pagecache.py -d or with inode2filename --format=textcache) :param inode_data_file: a file object (e.g. created with open or StringIO). Lifetime: inode_data_file is only used during the construction of the object. """ self._inode_table = Inode2Filename.build_inode_lookup_table(inode_data_file) @classmethod def new_from_filename(cls, textcache_filename: str) -> 'Inode2Filename': """ Create an Inode2Filename that reads cached inode from a file saved earlier (e.g. with pagecache.py -d or with inode2filename --format=textcache) :param textcache_filename: path to textcache """ with open(textcache_filename) as inode_data_file: return cls(inode_data_file) @staticmethod def build_inode_lookup_table(inode_data_file: TextIO) -> Dict[Tuple[int, int], Tuple[str, str]]: """ :return: map { (device_int, inode_int) -> (filename_str, size_str) } """ inode2filename = {} for line in inode_data_file: # stat -c "%d %i %s %n # device number, inode number, total size in bytes, file name result = re.match('([0-9]+)d? ([0-9]+) -?([0-9]+) (.*)', line) if result: inode2filename[(int(result.group(1)), int(result.group(2)))] = \ (result.group(4), result.group(3)) return inode2filename def resolve(self, dev_t: int, ino_t: int) -> Optional[str]: """ Return a filename (str) from a (dev_t, ino_t) inode pair. Returns None if the lookup fails. """ maybe_result = self._inode_table.get((dev_t, ino_t)) if not maybe_result: return None return maybe_result[0] # filename str def __len__(self) -> int: """ :return: the number of inode entries parsed from the file. """ return len(self._inode_table) def __repr__(self) -> str: """ :return: string representation for debugging/test failures. """ return "Inode2Filename%s" %(repr(self._inode_table)) # end of class. startop/scripts/iorap/lib/inode2filename_test.py 0 → 100755 +83 −0 Original line number Diff line number Diff line #!/usr/bin/env python3 # # Copyright 2019, The Android Open Source Project # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # """ Unit tests for inode2filename module. Install: $> sudo apt-get install python3-pytest ## OR $> pip install -U pytest See also https://docs.pytest.org/en/latest/getting-started.html Usage: $> ./inode2filename_test.py $> pytest inode2filename_test.py $> python -m pytest inode2filename_test.py See also https://docs.pytest.org/en/latest/usage.html """ # global imports from contextlib import contextmanager import io import shlex import sys import typing # pip imports import pytest # local imports from inode2filename import * def create_inode2filename(*contents): buf = io.StringIO() for c in contents: buf.write(c) buf.write("\n") buf.seek(0) i2f = Inode2Filename(buf) buf.close() return i2f def test_inode2filename(): a = create_inode2filename("") assert len(a) == 0 assert a.resolve(1, 2) == None a = create_inode2filename("1 2 3 foo.bar") assert len(a) == 1 assert a.resolve(1, 2) == "foo.bar" assert a.resolve(4, 5) == None a = create_inode2filename("1 2 3 foo.bar", "4 5 6 bar.baz") assert len(a) == 2 assert a.resolve(1, 2) == "foo.bar" assert a.resolve(4, 5) == "bar.baz" a = create_inode2filename("1567d 8910 -1 /a/b/c/", "4 5 6 bar.baz") assert len(a) == 2 assert a.resolve(1567, 8910) == "/a/b/c/" assert a.resolve(4, 5) == "bar.baz" if __name__ == '__main__': pytest.main() Loading
startop/scripts/iorap/compiler.py 0 → 100755 +244 −0 Original line number Diff line number Diff line #!/usr/bin/env python3 # # Copyright (C) 2019 The Android Open Source Project # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # # Dependencies: # # $> sudo apt-get install python3-pip # $> pip3 install --user protobuf sqlalchemy sqlite3 # import collections import optparse import os import re import sys from typing import Iterable from lib.inode2filename import Inode2Filename from generated.TraceFile_pb2 import * parent_dir_name = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) sys.path.append(parent_dir_name + "/trace_analyzer") from lib.trace2db import Trace2Db, MmFilemapAddToPageCache _PAGE_SIZE = 4096 # adb shell getconf PAGESIZE ## size of a memory page in bytes. class PageRun: """ Intermediate representation for a run of one or more pages. """ def __init__(self, device_number: int, inode: int, offset: int, length: int): self.device_number = device_number self.inode = inode self.offset = offset self.length = length def __str__(self): return "PageRun(device_number=%d, inode=%d, offset=%d, length=%d)" \ %(self.device_number, self.inode, self.offset, self.length) def debug_print(msg): #print(msg) pass UNDER_LAUNCH = False def page_cache_entries_to_runs(page_cache_entries: Iterable[MmFilemapAddToPageCache]): global _PAGE_SIZE runs = [ PageRun(device_number=pg_entry.dev, inode=pg_entry.ino, offset=pg_entry.ofs, length=_PAGE_SIZE) for pg_entry in page_cache_entries ] for r in runs: debug_print(r) print("Stats: Page runs totaling byte length: %d" %(len(runs) * _PAGE_SIZE)) return runs def optimize_page_runs(page_runs): new_entries = [] last_entry = None for pg_entry in page_runs: if last_entry: if pg_entry.device_number == last_entry.device_number and pg_entry.inode == last_entry.inode: # we are dealing with a run for the same exact file as a previous run. if pg_entry.offset == last_entry.offset + last_entry.length: # trivially contiguous entries. merge them together. last_entry.length += pg_entry.length continue # Default: Add the run without merging it to a previous run. last_entry = pg_entry new_entries.append(pg_entry) return new_entries def is_filename_matching_filter(file_name, filters=[]): """ Blacklist-style regular expression filters. :return: True iff file_name has an RE match in one of the filters. """ for filt in filters: res = re.search(filt, file_name) if res: return True return False def build_protobuf(page_runs, inode2filename, filters=[]): trace_file = TraceFile() trace_file_index = trace_file.index file_id_counter = 0 file_id_map = {} # filename -> id stats_length_total = 0 filename_stats = {} # filename -> total size skipped_inode_map = {} filtered_entry_map = {} # filename -> count for pg_entry in page_runs: fn = inode2filename.resolve(pg_entry.device_number, pg_entry.inode) if not fn: skipped_inode_map[pg_entry.inode] = skipped_inode_map.get(pg_entry.inode, 0) + 1 continue filename = fn if filters and not is_filename_matching_filter(filename, filters): filtered_entry_map[filename] = filtered_entry_map.get(filename, 0) + 1 continue file_id = file_id_map.get(filename) if not file_id: file_id = file_id_counter file_id_map[filename] = file_id_counter file_id_counter = file_id_counter + 1 file_index_entry = trace_file_index.entries.add() file_index_entry.id = file_id file_index_entry.file_name = filename # already in the file index, add the file entry. file_entry = trace_file.list.entries.add() file_entry.index_id = file_id file_entry.file_length = pg_entry.length stats_length_total += file_entry.file_length file_entry.file_offset = pg_entry.offset filename_stats[filename] = filename_stats.get(filename, 0) + file_entry.file_length for inode, count in skipped_inode_map.items(): print("WARNING: Skip inode %s because it's not in inode map (%d entries)" %(inode, count)) print("Stats: Sum of lengths %d" %(stats_length_total)) if filters: print("Filter: %d total files removed." %(len(filtered_entry_map))) for fn, count in filtered_entry_map.items(): print("Filter: File '%s' removed '%d' entries." %(fn, count)) for filename, file_size in filename_stats.items(): print("%s,%s" %(filename, file_size)) return trace_file def query_add_to_page_cache(trace2db: Trace2Db): # SELECT * FROM tbl ORDER BY id; return trace2db.session.query(MmFilemapAddToPageCache).order_by(MmFilemapAddToPageCache.id).all() def main(argv): parser = optparse.OptionParser(usage="Usage: %prog [options]", description="Compile systrace file into TraceFile.pb") parser.add_option('-i', dest='inode_data_file', metavar='FILE', help='Read cached inode data from a file saved earlier with pagecache.py -d') parser.add_option('-t', dest='trace_file', metavar='FILE', help='Path to systrace file (trace.html) that will be parsed') parser.add_option('--db', dest='sql_db', metavar='FILE', help='Path to intermediate sqlite3 database [default: in-memory].') parser.add_option('-f', dest='filter', action="append", default=[], help="Add file filter. All file entries not matching one of the filters are discarded.") parser.add_option('-l', dest='launch_lock', action="store_true", default=False, help="Exclude all events not inside launch_lock") parser.add_option('-o', dest='output_file', metavar='FILE', help='Output protobuf file') options, categories = parser.parse_args(argv[1:]) # TODO: OptionParser should have some flags to make these mandatory. if not options.inode_data_file: parser.error("-i is required") if not options.trace_file: parser.error("-t is required") if not options.output_file: parser.error("-o is required") if options.launch_lock: print("INFO: Launch lock flag (-l) enabled; filtering all events not inside launch_lock.") inode_table = Inode2Filename.new_from_filename(options.inode_data_file) trace_file = open(options.trace_file) sql_db_path = ":memory:" if options.sql_db: sql_db_path = options.sql_db trace2db = Trace2Db(sql_db_path) # Speed optimization: Skip any entries that aren't mm_filemap_add_to_pagecache. trace2db.set_raw_ftrace_entry_filter(\ lambda entry: entry['function'] == 'mm_filemap_add_to_page_cache') # TODO: parse multiple trace files here. parse_count = trace2db.parse_file_into_db(options.trace_file) mm_filemap_add_to_page_cache_rows = query_add_to_page_cache(trace2db) print("DONE. Parsed %d entries into sql db." %(len(mm_filemap_add_to_page_cache_rows))) page_runs = page_cache_entries_to_runs(mm_filemap_add_to_page_cache_rows) print("DONE. Converted %d entries" %(len(page_runs))) # TODO: flags to select optimizations. optimized_page_runs = optimize_page_runs(page_runs) print("DONE. Optimized down to %d entries" %(len(optimized_page_runs))) print("Build protobuf...") trace_file = build_protobuf(optimized_page_runs, inode_table, options.filter) print("Write protobuf to file...") output_file = open(options.output_file, 'wb') output_file.write(trace_file.SerializeToString()) output_file.close() print("DONE") # TODO: Silent running mode [no output except on error] for build runs. return 0 sys.exit(main(sys.argv))
startop/scripts/iorap/generated/TraceFile_pb2.py 0 → 100644 +259 −0 Original line number Diff line number Diff line # Generated by the protocol buffer compiler. DO NOT EDIT! # source: TraceFile.proto import sys _b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1')) from google.protobuf import descriptor as _descriptor from google.protobuf import message as _message from google.protobuf import reflection as _reflection from google.protobuf import symbol_database as _symbol_database from google.protobuf import descriptor_pb2 # @@protoc_insertion_point(imports) _sym_db = _symbol_database.Default() DESCRIPTOR = _descriptor.FileDescriptor( name='TraceFile.proto', package='iorap.serialize.proto', syntax='proto2', serialized_pb=_b('\n\x0fTraceFile.proto\x12\x15iorap.serialize.proto\"u\n\tTraceFile\x12\x34\n\x05index\x18\x01 \x02(\x0b\x32%.iorap.serialize.proto.TraceFileIndex\x12\x32\n\x04list\x18\x02 \x02(\x0b\x32$.iorap.serialize.proto.TraceFileList\"M\n\x0eTraceFileIndex\x12;\n\x07\x65ntries\x18\x01 \x03(\x0b\x32*.iorap.serialize.proto.TraceFileIndexEntry\"4\n\x13TraceFileIndexEntry\x12\n\n\x02id\x18\x01 \x02(\x03\x12\x11\n\tfile_name\x18\x02 \x02(\t\"G\n\rTraceFileList\x12\x36\n\x07\x65ntries\x18\x01 \x03(\x0b\x32%.iorap.serialize.proto.TraceFileEntry\"L\n\x0eTraceFileEntry\x12\x10\n\x08index_id\x18\x01 \x02(\x03\x12\x13\n\x0b\x66ile_offset\x18\x02 \x02(\x03\x12\x13\n\x0b\x66ile_length\x18\x03 \x02(\x03\x42\x1c\n\x18\x63om.google.android.iorapH\x03') ) _sym_db.RegisterFileDescriptor(DESCRIPTOR) _TRACEFILE = _descriptor.Descriptor( name='TraceFile', full_name='iorap.serialize.proto.TraceFile', filename=None, file=DESCRIPTOR, containing_type=None, fields=[ _descriptor.FieldDescriptor( name='index', full_name='iorap.serialize.proto.TraceFile.index', index=0, number=1, type=11, cpp_type=10, label=2, has_default_value=False, default_value=None, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), _descriptor.FieldDescriptor( name='list', full_name='iorap.serialize.proto.TraceFile.list', index=1, number=2, type=11, cpp_type=10, label=2, has_default_value=False, default_value=None, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), ], extensions=[ ], nested_types=[], enum_types=[ ], options=None, is_extendable=False, syntax='proto2', extension_ranges=[], oneofs=[ ], serialized_start=42, serialized_end=159, ) _TRACEFILEINDEX = _descriptor.Descriptor( name='TraceFileIndex', full_name='iorap.serialize.proto.TraceFileIndex', filename=None, file=DESCRIPTOR, containing_type=None, fields=[ _descriptor.FieldDescriptor( name='entries', full_name='iorap.serialize.proto.TraceFileIndex.entries', index=0, number=1, type=11, cpp_type=10, label=3, has_default_value=False, default_value=[], message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), ], extensions=[ ], nested_types=[], enum_types=[ ], options=None, is_extendable=False, syntax='proto2', extension_ranges=[], oneofs=[ ], serialized_start=161, serialized_end=238, ) _TRACEFILEINDEXENTRY = _descriptor.Descriptor( name='TraceFileIndexEntry', full_name='iorap.serialize.proto.TraceFileIndexEntry', filename=None, file=DESCRIPTOR, containing_type=None, fields=[ _descriptor.FieldDescriptor( name='id', full_name='iorap.serialize.proto.TraceFileIndexEntry.id', index=0, number=1, type=3, cpp_type=2, label=2, has_default_value=False, default_value=0, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), _descriptor.FieldDescriptor( name='file_name', full_name='iorap.serialize.proto.TraceFileIndexEntry.file_name', index=1, number=2, type=9, cpp_type=9, label=2, has_default_value=False, default_value=_b("").decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), ], extensions=[ ], nested_types=[], enum_types=[ ], options=None, is_extendable=False, syntax='proto2', extension_ranges=[], oneofs=[ ], serialized_start=240, serialized_end=292, ) _TRACEFILELIST = _descriptor.Descriptor( name='TraceFileList', full_name='iorap.serialize.proto.TraceFileList', filename=None, file=DESCRIPTOR, containing_type=None, fields=[ _descriptor.FieldDescriptor( name='entries', full_name='iorap.serialize.proto.TraceFileList.entries', index=0, number=1, type=11, cpp_type=10, label=3, has_default_value=False, default_value=[], message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), ], extensions=[ ], nested_types=[], enum_types=[ ], options=None, is_extendable=False, syntax='proto2', extension_ranges=[], oneofs=[ ], serialized_start=294, serialized_end=365, ) _TRACEFILEENTRY = _descriptor.Descriptor( name='TraceFileEntry', full_name='iorap.serialize.proto.TraceFileEntry', filename=None, file=DESCRIPTOR, containing_type=None, fields=[ _descriptor.FieldDescriptor( name='index_id', full_name='iorap.serialize.proto.TraceFileEntry.index_id', index=0, number=1, type=3, cpp_type=2, label=2, has_default_value=False, default_value=0, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), _descriptor.FieldDescriptor( name='file_offset', full_name='iorap.serialize.proto.TraceFileEntry.file_offset', index=1, number=2, type=3, cpp_type=2, label=2, has_default_value=False, default_value=0, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), _descriptor.FieldDescriptor( name='file_length', full_name='iorap.serialize.proto.TraceFileEntry.file_length', index=2, number=3, type=3, cpp_type=2, label=2, has_default_value=False, default_value=0, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), ], extensions=[ ], nested_types=[], enum_types=[ ], options=None, is_extendable=False, syntax='proto2', extension_ranges=[], oneofs=[ ], serialized_start=367, serialized_end=443, ) _TRACEFILE.fields_by_name['index'].message_type = _TRACEFILEINDEX _TRACEFILE.fields_by_name['list'].message_type = _TRACEFILELIST _TRACEFILEINDEX.fields_by_name['entries'].message_type = _TRACEFILEINDEXENTRY _TRACEFILELIST.fields_by_name['entries'].message_type = _TRACEFILEENTRY DESCRIPTOR.message_types_by_name['TraceFile'] = _TRACEFILE DESCRIPTOR.message_types_by_name['TraceFileIndex'] = _TRACEFILEINDEX DESCRIPTOR.message_types_by_name['TraceFileIndexEntry'] = _TRACEFILEINDEXENTRY DESCRIPTOR.message_types_by_name['TraceFileList'] = _TRACEFILELIST DESCRIPTOR.message_types_by_name['TraceFileEntry'] = _TRACEFILEENTRY TraceFile = _reflection.GeneratedProtocolMessageType('TraceFile', (_message.Message,), dict( DESCRIPTOR = _TRACEFILE, __module__ = 'TraceFile_pb2' # @@protoc_insertion_point(class_scope:iorap.serialize.proto.TraceFile) )) _sym_db.RegisterMessage(TraceFile) TraceFileIndex = _reflection.GeneratedProtocolMessageType('TraceFileIndex', (_message.Message,), dict( DESCRIPTOR = _TRACEFILEINDEX, __module__ = 'TraceFile_pb2' # @@protoc_insertion_point(class_scope:iorap.serialize.proto.TraceFileIndex) )) _sym_db.RegisterMessage(TraceFileIndex) TraceFileIndexEntry = _reflection.GeneratedProtocolMessageType('TraceFileIndexEntry', (_message.Message,), dict( DESCRIPTOR = _TRACEFILEINDEXENTRY, __module__ = 'TraceFile_pb2' # @@protoc_insertion_point(class_scope:iorap.serialize.proto.TraceFileIndexEntry) )) _sym_db.RegisterMessage(TraceFileIndexEntry) TraceFileList = _reflection.GeneratedProtocolMessageType('TraceFileList', (_message.Message,), dict( DESCRIPTOR = _TRACEFILELIST, __module__ = 'TraceFile_pb2' # @@protoc_insertion_point(class_scope:iorap.serialize.proto.TraceFileList) )) _sym_db.RegisterMessage(TraceFileList) TraceFileEntry = _reflection.GeneratedProtocolMessageType('TraceFileEntry', (_message.Message,), dict( DESCRIPTOR = _TRACEFILEENTRY, __module__ = 'TraceFile_pb2' # @@protoc_insertion_point(class_scope:iorap.serialize.proto.TraceFileEntry) )) _sym_db.RegisterMessage(TraceFileEntry) DESCRIPTOR.has_options = True DESCRIPTOR._options = _descriptor._ParseOptions(descriptor_pb2.FileOptions(), _b('\n\030com.google.android.iorapH\003')) # @@protoc_insertion_point(module_scope)
startop/scripts/iorap/generated/codegen_protos 0 → 100755 +35 −0 Original line number Diff line number Diff line #!/bin/bash # # Copyright 2019, The Android Open Source Project # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" APROTOC="$(which aprotoc)" IORAP_SERIALIZE_DIR="${DIR}/../../../../../../system/iorap/src/serialize" IORAP_PROTOS=($IORAP_SERIALIZE_DIR/*.proto) if [[ $? -ne 0 ]]; then echo "Fatal: Missing aprotoc. Set APROTOC=... or lunch build/envsetup.sh?" >&2 exit 1 fi if ! [[ -d $IORAP_SERIALIZE_DIR ]]; then echo "Fatal: Directory '$IORAP_SERIALIZE_DIR' does not exist." >&2 exit 1 fi # codegen the .py files into the same directory as this script. echo "$APROTOC" --proto_path="$IORAP_SERIALIZE_DIR" --python_out="$DIR" "${IORAP_PROTOS[@]}" "$APROTOC" --proto_path="$IORAP_SERIALIZE_DIR" --python_out="$DIR" "${IORAP_PROTOS[@]}"
startop/scripts/iorap/lib/inode2filename.py 0 → 100644 +94 −0 Original line number Diff line number Diff line #!/usr/bin/env python3 # # Copyright (C) 2019 The Android Open Source Project # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # from typing import Any, Callable, Dict, Generic, Iterable, List, NamedTuple, TextIO, Tuple, TypeVar, Optional, Union, TextIO import re class Inode2Filename: """ Parses a text file of the format "uint(dev_t) uint(ino_t) int(file_size) string(filepath)\\n"* Lines not matching this format are ignored. """ def __init__(self, inode_data_file: TextIO): """ Create an Inode2Filename that reads cached inode from a file saved earlier (e.g. with pagecache.py -d or with inode2filename --format=textcache) :param inode_data_file: a file object (e.g. created with open or StringIO). Lifetime: inode_data_file is only used during the construction of the object. """ self._inode_table = Inode2Filename.build_inode_lookup_table(inode_data_file) @classmethod def new_from_filename(cls, textcache_filename: str) -> 'Inode2Filename': """ Create an Inode2Filename that reads cached inode from a file saved earlier (e.g. with pagecache.py -d or with inode2filename --format=textcache) :param textcache_filename: path to textcache """ with open(textcache_filename) as inode_data_file: return cls(inode_data_file) @staticmethod def build_inode_lookup_table(inode_data_file: TextIO) -> Dict[Tuple[int, int], Tuple[str, str]]: """ :return: map { (device_int, inode_int) -> (filename_str, size_str) } """ inode2filename = {} for line in inode_data_file: # stat -c "%d %i %s %n # device number, inode number, total size in bytes, file name result = re.match('([0-9]+)d? ([0-9]+) -?([0-9]+) (.*)', line) if result: inode2filename[(int(result.group(1)), int(result.group(2)))] = \ (result.group(4), result.group(3)) return inode2filename def resolve(self, dev_t: int, ino_t: int) -> Optional[str]: """ Return a filename (str) from a (dev_t, ino_t) inode pair. Returns None if the lookup fails. """ maybe_result = self._inode_table.get((dev_t, ino_t)) if not maybe_result: return None return maybe_result[0] # filename str def __len__(self) -> int: """ :return: the number of inode entries parsed from the file. """ return len(self._inode_table) def __repr__(self) -> str: """ :return: string representation for debugging/test failures. """ return "Inode2Filename%s" %(repr(self._inode_table)) # end of class.
startop/scripts/iorap/lib/inode2filename_test.py 0 → 100755 +83 −0 Original line number Diff line number Diff line #!/usr/bin/env python3 # # Copyright 2019, The Android Open Source Project # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # """ Unit tests for inode2filename module. Install: $> sudo apt-get install python3-pytest ## OR $> pip install -U pytest See also https://docs.pytest.org/en/latest/getting-started.html Usage: $> ./inode2filename_test.py $> pytest inode2filename_test.py $> python -m pytest inode2filename_test.py See also https://docs.pytest.org/en/latest/usage.html """ # global imports from contextlib import contextmanager import io import shlex import sys import typing # pip imports import pytest # local imports from inode2filename import * def create_inode2filename(*contents): buf = io.StringIO() for c in contents: buf.write(c) buf.write("\n") buf.seek(0) i2f = Inode2Filename(buf) buf.close() return i2f def test_inode2filename(): a = create_inode2filename("") assert len(a) == 0 assert a.resolve(1, 2) == None a = create_inode2filename("1 2 3 foo.bar") assert len(a) == 1 assert a.resolve(1, 2) == "foo.bar" assert a.resolve(4, 5) == None a = create_inode2filename("1 2 3 foo.bar", "4 5 6 bar.baz") assert len(a) == 2 assert a.resolve(1, 2) == "foo.bar" assert a.resolve(4, 5) == "bar.baz" a = create_inode2filename("1567d 8910 -1 /a/b/c/", "4 5 6 bar.baz") assert len(a) == 2 assert a.resolve(1567, 8910) == "/a/b/c/" assert a.resolve(4, 5) == "bar.baz" if __name__ == '__main__': pytest.main()
