startop: Add iorap compiler written in python

#!/usr/bin/env python3

#

# Copyright (C) 2019 The Android Open Source Project

#

# Licensed under the Apache License, Version 2.0 (the "License");

# you may not use this file except in compliance with the License.

# You may obtain a copy of the License at

#

#      http://www.apache.org/licenses/LICENSE-2.0

#

# Unless required by applicable law or agreed to in writing, software

# distributed under the License is distributed on an "AS IS" BASIS,

# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

# See the License for the specific language governing permissions and

# limitations under the License.

#

#

# Dependencies:

#

# $> sudo apt-get install python3-pip

# $> pip3 install --user protobuf sqlalchemy sqlite3

#

import collections

import optparse

import os

import re

import sys

from typing import Iterable

from lib.inode2filename import Inode2Filename

from generated.TraceFile_pb2 import *

parent_dir_name = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))

sys.path.append(parent_dir_name + "/trace_analyzer")

from lib.trace2db import Trace2Db, MmFilemapAddToPageCache

_PAGE_SIZE = 4096 # adb shell getconf PAGESIZE ## size of a memory page in bytes.

class PageRun:

  """

  Intermediate representation for a run of one or more pages.

  """

  def __init__(self, device_number: int, inode: int, offset: int, length: int):

    self.device_number = device_number

    self.inode = inode

    self.offset = offset

    self.length = length

  def __str__(self):

    return "PageRun(device_number=%d, inode=%d, offset=%d, length=%d)" \

        %(self.device_number, self.inode, self.offset, self.length)

def debug_print(msg):

  #print(msg)

  pass

UNDER_LAUNCH = False

def page_cache_entries_to_runs(page_cache_entries: Iterable[MmFilemapAddToPageCache]):

  global _PAGE_SIZE

  runs = [

      PageRun(device_number=pg_entry.dev, inode=pg_entry.ino, offset=pg_entry.ofs,

              length=_PAGE_SIZE)

        for pg_entry in page_cache_entries

  ]

  for r in runs:

    debug_print(r)

  print("Stats: Page runs totaling byte length: %d" %(len(runs) * _PAGE_SIZE))

  return runs

def optimize_page_runs(page_runs):

  new_entries = []

  last_entry = None

  for pg_entry in page_runs:

    if last_entry:

      if pg_entry.device_number == last_entry.device_number and pg_entry.inode == last_entry.inode:

        # we are dealing with a run for the same exact file as a previous run.

        if pg_entry.offset == last_entry.offset + last_entry.length:

          # trivially contiguous entries. merge them together.

          last_entry.length += pg_entry.length

          continue

    # Default: Add the run without merging it to a previous run.

    last_entry = pg_entry

    new_entries.append(pg_entry)

  return new_entries

def is_filename_matching_filter(file_name, filters=[]):

  """

  Blacklist-style regular expression filters.

  :return: True iff file_name has an RE match in one of the filters.

  """

  for filt in filters:

    res = re.search(filt, file_name)

    if res:

      return True

  return False

def build_protobuf(page_runs, inode2filename, filters=[]):

  trace_file = TraceFile()

  trace_file_index = trace_file.index

  file_id_counter = 0

  file_id_map = {} # filename -> id

  stats_length_total = 0

  filename_stats = {} # filename -> total size

  skipped_inode_map = {}

  filtered_entry_map = {} # filename -> count

  for pg_entry in page_runs:

    fn = inode2filename.resolve(pg_entry.device_number, pg_entry.inode)

    if not fn:

      skipped_inode_map[pg_entry.inode] = skipped_inode_map.get(pg_entry.inode, 0) + 1

      continue

    filename = fn

    if filters and not is_filename_matching_filter(filename, filters):

      filtered_entry_map[filename] = filtered_entry_map.get(filename, 0) + 1

      continue

    file_id = file_id_map.get(filename)

    if not file_id:

      file_id = file_id_counter

      file_id_map[filename] = file_id_counter

      file_id_counter = file_id_counter + 1

      file_index_entry = trace_file_index.entries.add()

      file_index_entry.id = file_id

      file_index_entry.file_name = filename

    # already in the file index, add the file entry.

    file_entry = trace_file.list.entries.add()

    file_entry.index_id = file_id

    file_entry.file_length = pg_entry.length

    stats_length_total += file_entry.file_length

    file_entry.file_offset = pg_entry.offset

    filename_stats[filename] = filename_stats.get(filename, 0) + file_entry.file_length

  for inode, count in skipped_inode_map.items():

    print("WARNING: Skip inode %s because it's not in inode map (%d entries)" %(inode, count))

  print("Stats: Sum of lengths %d" %(stats_length_total))

  if filters:

    print("Filter: %d total files removed." %(len(filtered_entry_map)))

    for fn, count in filtered_entry_map.items():

      print("Filter: File '%s' removed '%d' entries." %(fn, count))

  for filename, file_size in filename_stats.items():

    print("%s,%s" %(filename, file_size))

  return trace_file

def query_add_to_page_cache(trace2db: Trace2Db):

  # SELECT * FROM tbl ORDER BY id;

  return trace2db.session.query(MmFilemapAddToPageCache).order_by(MmFilemapAddToPageCache.id).all()

def main(argv):

  parser = optparse.OptionParser(usage="Usage: %prog [options]", description="Compile systrace file into TraceFile.pb")

  parser.add_option('-i', dest='inode_data_file', metavar='FILE',

                    help='Read cached inode data from a file saved earlier with pagecache.py -d')

  parser.add_option('-t', dest='trace_file', metavar='FILE',

                    help='Path to systrace file (trace.html) that will be parsed')

  parser.add_option('--db', dest='sql_db', metavar='FILE',

                    help='Path to intermediate sqlite3 database [default: in-memory].')

  parser.add_option('-f', dest='filter', action="append", default=[],

                    help="Add file filter. All file entries not matching one of the filters are discarded.")

  parser.add_option('-l', dest='launch_lock', action="store_true", default=False,

                    help="Exclude all events not inside launch_lock")

  parser.add_option('-o', dest='output_file', metavar='FILE',

                    help='Output protobuf file')

  options, categories = parser.parse_args(argv[1:])

  # TODO: OptionParser should have some flags to make these mandatory.

  if not options.inode_data_file:

    parser.error("-i is required")

  if not options.trace_file:

    parser.error("-t is required")

  if not options.output_file:

    parser.error("-o is required")

  if options.launch_lock:

    print("INFO: Launch lock flag (-l) enabled; filtering all events not inside launch_lock.")

  inode_table = Inode2Filename.new_from_filename(options.inode_data_file)

  trace_file = open(options.trace_file)

  sql_db_path = ":memory:"

  if options.sql_db:

    sql_db_path = options.sql_db

  trace2db = Trace2Db(sql_db_path)

  # Speed optimization: Skip any entries that aren't mm_filemap_add_to_pagecache.

  trace2db.set_raw_ftrace_entry_filter(\

      lambda entry: entry['function'] == 'mm_filemap_add_to_page_cache')

  # TODO: parse multiple trace files here.

  parse_count = trace2db.parse_file_into_db(options.trace_file)

  mm_filemap_add_to_page_cache_rows = query_add_to_page_cache(trace2db)

  print("DONE. Parsed %d entries into sql db." %(len(mm_filemap_add_to_page_cache_rows)))

  page_runs = page_cache_entries_to_runs(mm_filemap_add_to_page_cache_rows)

  print("DONE. Converted %d entries" %(len(page_runs)))

  # TODO: flags to select optimizations.

  optimized_page_runs = optimize_page_runs(page_runs)

  print("DONE. Optimized down to %d entries" %(len(optimized_page_runs)))

  print("Build protobuf...")

  trace_file = build_protobuf(optimized_page_runs, inode_table, options.filter)

  print("Write protobuf to file...")

  output_file = open(options.output_file, 'wb')

  output_file.write(trace_file.SerializeToString())

  output_file.close()

  print("DONE")

  # TODO: Silent running mode [no output except on error] for build runs.

  return 0

sys.exit(main(sys.argv))

# Generated by the protocol buffer compiler.  DO NOT EDIT!

# source: TraceFile.proto

import sys

_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1'))

from google.protobuf import descriptor as _descriptor

from google.protobuf import message as _message

from google.protobuf import reflection as _reflection

from google.protobuf import symbol_database as _symbol_database

from google.protobuf import descriptor_pb2

# @@protoc_insertion_point(imports)

_sym_db = _symbol_database.Default()

DESCRIPTOR = _descriptor.FileDescriptor(

  name='TraceFile.proto',

  package='iorap.serialize.proto',

  syntax='proto2',

  serialized_pb=_b('\n\x0fTraceFile.proto\x12\x15iorap.serialize.proto\"u\n\tTraceFile\x12\x34\n\x05index\x18\x01 \x02(\x0b\x32%.iorap.serialize.proto.TraceFileIndex\x12\x32\n\x04list\x18\x02 \x02(\x0b\x32$.iorap.serialize.proto.TraceFileList\"M\n\x0eTraceFileIndex\x12;\n\x07\x65ntries\x18\x01 \x03(\x0b\x32*.iorap.serialize.proto.TraceFileIndexEntry\"4\n\x13TraceFileIndexEntry\x12\n\n\x02id\x18\x01 \x02(\x03\x12\x11\n\tfile_name\x18\x02 \x02(\t\"G\n\rTraceFileList\x12\x36\n\x07\x65ntries\x18\x01 \x03(\x0b\x32%.iorap.serialize.proto.TraceFileEntry\"L\n\x0eTraceFileEntry\x12\x10\n\x08index_id\x18\x01 \x02(\x03\x12\x13\n\x0b\x66ile_offset\x18\x02 \x02(\x03\x12\x13\n\x0b\x66ile_length\x18\x03 \x02(\x03\x42\x1c\n\x18\x63om.google.android.iorapH\x03')

)

_sym_db.RegisterFileDescriptor(DESCRIPTOR)

_TRACEFILE = _descriptor.Descriptor(

  name='TraceFile',

  full_name='iorap.serialize.proto.TraceFile',

  filename=None,

  file=DESCRIPTOR,

  containing_type=None,

  fields=[

    _descriptor.FieldDescriptor(

      name='index', full_name='iorap.serialize.proto.TraceFile.index', index=0,

      number=1, type=11, cpp_type=10, label=2,

      has_default_value=False, default_value=None,

      message_type=None, enum_type=None, containing_type=None,

      is_extension=False, extension_scope=None,

      options=None),

    _descriptor.FieldDescriptor(

      name='list', full_name='iorap.serialize.proto.TraceFile.list', index=1,

      number=2, type=11, cpp_type=10, label=2,

      has_default_value=False, default_value=None,

      message_type=None, enum_type=None, containing_type=None,

      is_extension=False, extension_scope=None,

      options=None),

  ],

  extensions=[

  ],

  nested_types=[],

  enum_types=[

  ],

  options=None,

  is_extendable=False,

  syntax='proto2',

  extension_ranges=[],

  oneofs=[

  ],

  serialized_start=42,

  serialized_end=159,

)

_TRACEFILEINDEX = _descriptor.Descriptor(

  name='TraceFileIndex',

  full_name='iorap.serialize.proto.TraceFileIndex',

  filename=None,

  file=DESCRIPTOR,

  containing_type=None,

  fields=[

    _descriptor.FieldDescriptor(

      name='entries', full_name='iorap.serialize.proto.TraceFileIndex.entries', index=0,

      number=1, type=11, cpp_type=10, label=3,

      has_default_value=False, default_value=[],

      message_type=None, enum_type=None, containing_type=None,

      is_extension=False, extension_scope=None,

      options=None),

  ],

  extensions=[

  ],

  nested_types=[],

  enum_types=[

  ],

  options=None,

  is_extendable=False,

  syntax='proto2',

  extension_ranges=[],

  oneofs=[

  ],

  serialized_start=161,

  serialized_end=238,

)

_TRACEFILEINDEXENTRY = _descriptor.Descriptor(

  name='TraceFileIndexEntry',

  full_name='iorap.serialize.proto.TraceFileIndexEntry',

  filename=None,

  file=DESCRIPTOR,

  containing_type=None,

  fields=[

    _descriptor.FieldDescriptor(

      name='id', full_name='iorap.serialize.proto.TraceFileIndexEntry.id', index=0,

      number=1, type=3, cpp_type=2, label=2,

      has_default_value=False, default_value=0,

      message_type=None, enum_type=None, containing_type=None,

      is_extension=False, extension_scope=None,

      options=None),

    _descriptor.FieldDescriptor(

      name='file_name', full_name='iorap.serialize.proto.TraceFileIndexEntry.file_name', index=1,

      number=2, type=9, cpp_type=9, label=2,

      has_default_value=False, default_value=_b("").decode('utf-8'),

      message_type=None, enum_type=None, containing_type=None,

      is_extension=False, extension_scope=None,

      options=None),

  ],

  extensions=[

  ],

  nested_types=[],

  enum_types=[

  ],

  options=None,

  is_extendable=False,

  syntax='proto2',

  extension_ranges=[],

  oneofs=[

  ],

  serialized_start=240,

  serialized_end=292,

)

_TRACEFILELIST = _descriptor.Descriptor(

  name='TraceFileList',

  full_name='iorap.serialize.proto.TraceFileList',

  filename=None,

  file=DESCRIPTOR,

  containing_type=None,

  fields=[

    _descriptor.FieldDescriptor(

      name='entries', full_name='iorap.serialize.proto.TraceFileList.entries', index=0,

      number=1, type=11, cpp_type=10, label=3,

      has_default_value=False, default_value=[],

      message_type=None, enum_type=None, containing_type=None,

      is_extension=False, extension_scope=None,

      options=None),

  ],

  extensions=[

  ],

  nested_types=[],

  enum_types=[

  ],

  options=None,

  is_extendable=False,

  syntax='proto2',

  extension_ranges=[],

  oneofs=[

  ],

  serialized_start=294,

  serialized_end=365,

)

_TRACEFILEENTRY = _descriptor.Descriptor(

  name='TraceFileEntry',

  full_name='iorap.serialize.proto.TraceFileEntry',

  filename=None,

  file=DESCRIPTOR,

  containing_type=None,

  fields=[

    _descriptor.FieldDescriptor(

      name='index_id', full_name='iorap.serialize.proto.TraceFileEntry.index_id', index=0,

      number=1, type=3, cpp_type=2, label=2,

      has_default_value=False, default_value=0,

      message_type=None, enum_type=None, containing_type=None,

      is_extension=False, extension_scope=None,

      options=None),

    _descriptor.FieldDescriptor(

      name='file_offset', full_name='iorap.serialize.proto.TraceFileEntry.file_offset', index=1,

      number=2, type=3, cpp_type=2, label=2,

      has_default_value=False, default_value=0,

      message_type=None, enum_type=None, containing_type=None,

      is_extension=False, extension_scope=None,

      options=None),

    _descriptor.FieldDescriptor(

      name='file_length', full_name='iorap.serialize.proto.TraceFileEntry.file_length', index=2,

      number=3, type=3, cpp_type=2, label=2,

      has_default_value=False, default_value=0,

      message_type=None, enum_type=None, containing_type=None,

      is_extension=False, extension_scope=None,

      options=None),

  ],

  extensions=[

  ],

  nested_types=[],

  enum_types=[

  ],

  options=None,

  is_extendable=False,

  syntax='proto2',

  extension_ranges=[],

  oneofs=[

  ],

  serialized_start=367,

  serialized_end=443,

)

_TRACEFILE.fields_by_name['index'].message_type = _TRACEFILEINDEX

_TRACEFILE.fields_by_name['list'].message_type = _TRACEFILELIST

_TRACEFILEINDEX.fields_by_name['entries'].message_type = _TRACEFILEINDEXENTRY

_TRACEFILELIST.fields_by_name['entries'].message_type = _TRACEFILEENTRY

DESCRIPTOR.message_types_by_name['TraceFile'] = _TRACEFILE

DESCRIPTOR.message_types_by_name['TraceFileIndex'] = _TRACEFILEINDEX

DESCRIPTOR.message_types_by_name['TraceFileIndexEntry'] = _TRACEFILEINDEXENTRY

DESCRIPTOR.message_types_by_name['TraceFileList'] = _TRACEFILELIST

DESCRIPTOR.message_types_by_name['TraceFileEntry'] = _TRACEFILEENTRY

TraceFile = _reflection.GeneratedProtocolMessageType('TraceFile', (_message.Message,), dict(

  DESCRIPTOR = _TRACEFILE,

  __module__ = 'TraceFile_pb2'

  # @@protoc_insertion_point(class_scope:iorap.serialize.proto.TraceFile)

  ))

_sym_db.RegisterMessage(TraceFile)

TraceFileIndex = _reflection.GeneratedProtocolMessageType('TraceFileIndex', (_message.Message,), dict(

  DESCRIPTOR = _TRACEFILEINDEX,

  __module__ = 'TraceFile_pb2'

  # @@protoc_insertion_point(class_scope:iorap.serialize.proto.TraceFileIndex)

  ))

_sym_db.RegisterMessage(TraceFileIndex)

TraceFileIndexEntry = _reflection.GeneratedProtocolMessageType('TraceFileIndexEntry', (_message.Message,), dict(

  DESCRIPTOR = _TRACEFILEINDEXENTRY,

  __module__ = 'TraceFile_pb2'

  # @@protoc_insertion_point(class_scope:iorap.serialize.proto.TraceFileIndexEntry)

  ))

_sym_db.RegisterMessage(TraceFileIndexEntry)

TraceFileList = _reflection.GeneratedProtocolMessageType('TraceFileList', (_message.Message,), dict(

  DESCRIPTOR = _TRACEFILELIST,

  __module__ = 'TraceFile_pb2'

  # @@protoc_insertion_point(class_scope:iorap.serialize.proto.TraceFileList)

  ))

_sym_db.RegisterMessage(TraceFileList)

TraceFileEntry = _reflection.GeneratedProtocolMessageType('TraceFileEntry', (_message.Message,), dict(

  DESCRIPTOR = _TRACEFILEENTRY,

  __module__ = 'TraceFile_pb2'

  # @@protoc_insertion_point(class_scope:iorap.serialize.proto.TraceFileEntry)

  ))

_sym_db.RegisterMessage(TraceFileEntry)

DESCRIPTOR.has_options = True

DESCRIPTOR._options = _descriptor._ParseOptions(descriptor_pb2.FileOptions(), _b('\n\030com.google.android.iorapH\003'))

# @@protoc_insertion_point(module_scope)

#!/bin/bash

#

# Copyright 2019, The Android Open Source Project

#

# Licensed under the Apache License, Version 2.0 (the "License");

# you may not use this file except in compliance with the License.

# You may obtain a copy of the License at

#

#     http://www.apache.org/licenses/LICENSE-2.0

#

# Unless required by applicable law or agreed to in writing, software

# distributed under the License is distributed on an "AS IS" BASIS,

# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

# See the License for the specific language governing permissions and

# limitations under the License.

DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"

APROTOC="$(which aprotoc)"

IORAP_SERIALIZE_DIR="${DIR}/../../../../../../system/iorap/src/serialize"

IORAP_PROTOS=($IORAP_SERIALIZE_DIR/*.proto)

if [[ $? -ne 0 ]]; then

  echo "Fatal: Missing aprotoc. Set APROTOC=... or lunch build/envsetup.sh?" >&2

  exit 1

fi

if ! [[ -d $IORAP_SERIALIZE_DIR ]]; then

  echo "Fatal: Directory '$IORAP_SERIALIZE_DIR' does not exist." >&2

  exit 1

fi

# codegen the .py files into the same directory as this script.

echo "$APROTOC" --proto_path="$IORAP_SERIALIZE_DIR" --python_out="$DIR" "${IORAP_PROTOS[@]}"

"$APROTOC" --proto_path="$IORAP_SERIALIZE_DIR" --python_out="$DIR" "${IORAP_PROTOS[@]}"

#!/usr/bin/env python3

#

# Copyright (C) 2019 The Android Open Source Project

#

# Licensed under the Apache License, Version 2.0 (the "License");

# you may not use this file except in compliance with the License.

# You may obtain a copy of the License at

#

#      http://www.apache.org/licenses/LICENSE-2.0

#

# Unless required by applicable law or agreed to in writing, software

# distributed under the License is distributed on an "AS IS" BASIS,

# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

# See the License for the specific language governing permissions and

# limitations under the License.

#

from typing import Any, Callable, Dict, Generic, Iterable, List, NamedTuple, TextIO, Tuple, TypeVar, Optional, Union, TextIO

import re

class Inode2Filename:

  """

  Parses a text file of the format

     "uint(dev_t) uint(ino_t) int(file_size) string(filepath)\\n"*

  Lines not matching this format are ignored.

  """

  def __init__(self, inode_data_file: TextIO):

    """

    Create an Inode2Filename that reads cached inode from a file saved earlier

    (e.g. with pagecache.py -d or with inode2filename --format=textcache)

    :param inode_data_file: a file object (e.g. created with open or StringIO).

    Lifetime: inode_data_file is only used during the construction of the object.

    """

    self._inode_table = Inode2Filename.build_inode_lookup_table(inode_data_file)

  @classmethod

  def new_from_filename(cls, textcache_filename: str) -> 'Inode2Filename':

    """

    Create an Inode2Filename that reads cached inode from a file saved earlier

    (e.g. with pagecache.py -d or with inode2filename --format=textcache)

    :param textcache_filename: path to textcache

    """

    with open(textcache_filename) as inode_data_file:

      return cls(inode_data_file)

  @staticmethod

  def build_inode_lookup_table(inode_data_file: TextIO) -> Dict[Tuple[int, int], Tuple[str, str]]:

    """

    :return: map { (device_int, inode_int) -> (filename_str, size_str) }

    """

    inode2filename = {}

    for line in inode_data_file:

      # stat -c "%d %i %s %n

      # device number, inode number, total size in bytes, file name

      result = re.match('([0-9]+)d? ([0-9]+) -?([0-9]+) (.*)', line)

      if result:

        inode2filename[(int(result.group(1)), int(result.group(2)))] = \

            (result.group(4), result.group(3))

    return inode2filename

  def resolve(self, dev_t: int, ino_t: int) -> Optional[str]:

    """

    Return a filename (str) from a (dev_t, ino_t) inode pair.

    Returns None if the lookup fails.

    """

    maybe_result = self._inode_table.get((dev_t, ino_t))

    if not maybe_result:

      return None

    return maybe_result[0] # filename str

  def __len__(self) -> int:

    """

    :return: the number of inode entries parsed from the file.

    """

    return len(self._inode_table)

  def __repr__(self) -> str:

    """

    :return: string representation for debugging/test failures.

    """

    return "Inode2Filename%s" %(repr(self._inode_table))

  # end of class.

#!/usr/bin/env python3

#

# Copyright 2019, The Android Open Source Project

#

# Licensed under the Apache License, Version 2.0 (the "License");

# you may not use this file except in compliance with the License.

# You may obtain a copy of the License at

#

#     http://www.apache.org/licenses/LICENSE-2.0

#

# Unless required by applicable law or agreed to in writing, software

# distributed under the License is distributed on an "AS IS" BASIS,

# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

# See the License for the specific language governing permissions and

# limitations under the License.

#

"""

Unit tests for inode2filename module.

Install:

  $> sudo apt-get install python3-pytest   ##  OR

  $> pip install -U pytest

See also https://docs.pytest.org/en/latest/getting-started.html

Usage:

  $> ./inode2filename_test.py

  $> pytest inode2filename_test.py

  $> python -m pytest inode2filename_test.py

See also https://docs.pytest.org/en/latest/usage.html

"""

# global imports