Merge "startop: Make app_startup_runner.py --compiler-filter force compilation"

#!/usr/bin/env python3

#

# Copyright 2018, The Android Open Source Project

#

# Licensed under the Apache License, Version 2.0 (the "License");

# you may not use this file except in compliance with the License.

# You may obtain a copy of the License at

#

#     http://www.apache.org/licenses/LICENSE-2.0

#

# Unless required by applicable law or agreed to in writing, software

# distributed under the License is distributed on an "AS IS" BASIS,

# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

# See the License for the specific language governing permissions and

# limitations under the License.

#

#

# Query the current compiler filter for an application by its package name.

# (By parsing the results of the 'adb shell dumpsys package $package' command).

# The output is a string "$compilation_filter $compilation_reason $isa".

#

# See --help for more details.

#

# -----------------------------------

#

# Sample usage:

#

# $> ./query_compiler_filter.py --package com.google.android.calculator

# speed-profile unknown arm64

#

import argparse

import sys

import re

# TODO: refactor this with a common library file with analyze_metrics.py

import app_startup_runner

from app_startup_runner import _debug_print

from app_startup_runner import execute_arbitrary_command

from typing import List, NamedTuple, Iterable

_DEBUG_FORCE = None  # Ignore -d/--debug if this is not none.

def parse_options(argv: List[str] = None):

  """Parse command line arguments and return an argparse Namespace object."""

  parser = argparse.ArgumentParser(description="Query the compiler filter for a package.")

  # argparse considers args starting with - and -- optional in --help, even though required=True.

  # by using a named argument group --help will clearly say that it's required instead of optional.

  required_named = parser.add_argument_group('required named arguments')

  required_named.add_argument('-p', '--package', action='store', dest='package', help='package of the application', required=True)

  # optional arguments

  # use a group here to get the required arguments to appear 'above' the optional arguments in help.

  optional_named = parser.add_argument_group('optional named arguments')

  optional_named.add_argument('-i', '--isa', '--instruction-set', action='store', dest='instruction_set', help='which instruction set to select. defaults to the first one available if not specified.', choices=('arm64', 'arm', 'x86_64', 'x86'))

  optional_named.add_argument('-s', '--simulate', dest='simulate', action='store_true', help='Print which commands will run, but don\'t run the apps')

  optional_named.add_argument('-d', '--debug', dest='debug', action='store_true', help='Add extra debugging output')

  return parser.parse_args(argv)

def remote_dumpsys_package(package: str, simulate: bool) -> str:

  # --simulate is used for interactive debugging/development, but also for the unit test.

  if simulate:

    return """

Dexopt state:

  [%s]

    path: /data/app/%s-D7s8PLidqqEq7Jc7UH_a5A==/base.apk

      arm64: [status=speed-profile] [reason=unknown]

    path: /data/app/%s-D7s8PLidqqEq7Jc7UH_a5A==/base.apk

      arm: [status=speed] [reason=first-boot]

    path: /data/app/%s-D7s8PLidqqEq7Jc7UH_a5A==/base.apk

      x86: [status=quicken] [reason=install]

""" %(package, package, package, package)

  code, res = execute_arbitrary_command(['adb', 'shell', 'dumpsys', 'package', package], simulate=False, timeout=5)

  if code:

    return res

  else:

    raise AssertionError("Failed to dumpsys package, errors = %s", res)

ParseTree = NamedTuple('ParseTree', [('label', str), ('children', List['ParseTree'])])

DexoptState = ParseTree # With the Dexopt state: label

ParseResult = NamedTuple('ParseResult', [('remainder', List[str]), ('tree', ParseTree)])

def find_parse_subtree(parse_tree: ParseTree, match_regex: str) -> ParseTree:

  if re.match(match_regex, parse_tree.label):

    return parse_tree

  for node in parse_tree.children:

    res = find_parse_subtree(node, match_regex)

    if res:

      return res

  return None

def find_parse_children(parse_tree: ParseTree, match_regex: str) -> Iterable[ParseTree]:

  for node in parse_tree.children:

    if re.match(match_regex, node.label):

      yield node

def parse_tab_subtree(label: str, str_lines: List[str], separator=' ', indent=-1) -> ParseResult:

  children = []

  get_indent_level = lambda line: len(line) - len(line.lstrip())

  line_num = 0

  keep_going = True

  while keep_going:

    keep_going = False

    for line_num in range(len(str_lines)):

      line = str_lines[line_num]

      current_indent = get_indent_level(line)

      _debug_print("INDENT=%d, LINE=%s" %(current_indent, line))

      current_label = line.lstrip()

      # skip empty lines

      if line.lstrip() == "":

        continue

      if current_indent > indent:

        parse_result = parse_tab_subtree(current_label, str_lines[line_num+1::], separator, current_indent)

        str_lines = parse_result.remainder

        children.append(parse_result.tree)

        keep_going = True

      else:

        # current_indent <= indent

        keep_going = False

      break

  new_remainder = str_lines[line_num::]

  _debug_print("NEW REMAINDER: ", new_remainder)

  parse_tree = ParseTree(label, children)

  return ParseResult(new_remainder, parse_tree)

def parse_tab_tree(str_tree: str, separator=' ', indentation_level=-1) -> ParseTree:

  label = None

  lst = []

  line_num = 0

  line_lst = str_tree.split("\n")

  return parse_tab_subtree("", line_lst, separator, indentation_level).tree

def parse_dexopt_state(dumpsys_tree: ParseTree) -> DexoptState:

  res = find_parse_subtree(dumpsys_tree, "Dexopt(\s+)state[:]?")

  if not res:

    raise AssertionError("Could not find the Dexopt state")

  return res

def find_first_compiler_filter(dexopt_state: DexoptState, package: str, instruction_set: str) -> str:

  lst = find_all_compiler_filters(dexopt_state, package)

  _debug_print("all compiler filters: ", lst)

  for compiler_filter_info in lst:

    if not instruction_set:

      return compiler_filter_info

    if compiler_filter_info.isa == instruction_set:

      return compiler_filter_info

  return None

CompilerFilterInfo = NamedTuple('CompilerFilterInfo', [('isa', str), ('status', str), ('reason', str)])

def find_all_compiler_filters(dexopt_state: DexoptState, package: str) -> List[CompilerFilterInfo]:

  lst = []

  package_tree = find_parse_subtree(dexopt_state, re.escape("[%s]" %package))

  if not package_tree:

    raise AssertionError("Could not find any package subtree for package %s" %(package))

  _debug_print("package tree: ", package_tree)

  for path_tree in find_parse_children(package_tree, "path: "):

    _debug_print("path tree: ", path_tree)

    matchre = re.compile("([^:]+):\s+\[status=([^\]]+)\]\s+\[reason=([^\]]+)\]")

    for isa_node in find_parse_children(path_tree, matchre):

      matches = re.match(matchre, isa_node.label).groups()

      info = CompilerFilterInfo(*matches)

      lst.append(info)

  return lst

def main() -> int:

  opts = parse_options()

  app_startup_runner._debug = opts.debug

  if _DEBUG_FORCE is not None:

    app_startup_runner._debug = _DEBUG_FORCE

  _debug_print("parsed options: ", opts)

  # Note: This can often 'fail' if the package isn't actually installed.

  package_dumpsys = remote_dumpsys_package(opts.package, opts.simulate)

  _debug_print("package dumpsys: ", package_dumpsys)

  dumpsys_parse_tree = parse_tab_tree(package_dumpsys, package_dumpsys)

  _debug_print("parse tree: ", dumpsys_parse_tree)

  dexopt_state = parse_dexopt_state(dumpsys_parse_tree)

  filter = find_first_compiler_filter(dexopt_state, opts.package, opts.instruction_set)

  if filter:

    print(filter.status, end=' ')

    print(filter.reason, end=' ')

    print(filter.isa)

  else:

    print("ERROR: Could not find any compiler-filter for package %s, isa %s" %(opts.package, opts.instruction_set), file=sys.stderr)

    return 1

  return 0

if __name__ == '__main__':

  sys.exit(main())

#!/usr/bin/env python3

#

# Copyright 2018, The Android Open Source Project

#

# Licensed under the Apache License, Version 2.0 (the "License");

# you may not use this file except in compliance with the License.

# You may obtain a copy of the License at

#

#     http://www.apache.org/licenses/LICENSE-2.0

#

# Unless required by applicable law or agreed to in writing, software

# distributed under the License is distributed on an "AS IS" BASIS,

# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

# See the License for the specific language governing permissions and

# limitations under the License.

#

"""

Unit tests for the query_compiler_filter.py script.

Install:

  $> sudo apt-get install python3-pytest   ##  OR

  $> pip install -U pytest

See also https://docs.pytest.org/en/latest/getting-started.html

Usage:

  $> ./query_compiler_filter.py

  $> pytest query_compiler_filter.py

  $> python -m pytest query_compiler_filter.py

See also https://docs.pytest.org/en/latest/usage.html

"""

# global imports

from contextlib import contextmanager

import io

import shlex

import sys

import typing

# pip imports

import pytest

# local imports

import query_compiler_filter as qcf

@contextmanager

def redirect_stdout_stderr():

  """Redirect stdout/stderr to a new StringIO for duration of context."""

  old_stdout = sys.stdout

  old_stderr = sys.stderr

  new_stdout = io.StringIO()

  sys.stdout = new_stdout

  new_stderr = io.StringIO()

  sys.stderr = new_stderr

  try:

    yield (new_stdout, new_stderr)

  finally:

    sys.stdout = old_stdout

    sys.stderr = old_stderr

    # Seek back to the beginning so we can read whatever was written into it.

    new_stdout.seek(0)

    new_stderr.seek(0)

@contextmanager

def replace_argv(argv):

  """ Temporarily replace argv for duration of this context."""

  old_argv = sys.argv

  sys.argv = [sys.argv[0]] + argv

  try:

    yield

  finally:

    sys.argv = old_argv

def exec_main(argv):

  """Run the query_compiler_filter main function with the provided arguments.

  Returns the stdout result when successful, assertion failure otherwise.

  """

  try:

    with redirect_stdout_stderr() as (the_stdout, the_stderr):

      with replace_argv(argv):

        code = qcf.main()

    assert 0 == code, the_stderr.readlines()

    all_lines = the_stdout.readlines()

    return "".join(all_lines)

  finally:

    the_stdout.close()

    the_stderr.close()

def test_query_compiler_filter():

  # no --instruction-set specified: provide whatever was the 'first' filter.

  assert exec_main(['--simulate',

                    '--package', 'com.google.android.apps.maps']) == \

      "speed-profile unknown arm64\n"

  # specifying an instruction set finds the exact compiler filter match.

  assert exec_main(['--simulate',

                    '--package', 'com.google.android.apps.maps',

                    '--instruction-set', 'arm64']) == \

      "speed-profile unknown arm64\n"

  assert exec_main(['--simulate',

                    '--package', 'com.google.android.apps.maps',

                    '--instruction-set', 'arm']) == \

      "speed first-boot arm\n"

  assert exec_main(['--simulate',

                    '--debug',

                    '--package', 'com.google.android.apps.maps',

                    '--instruction-set', 'x86']) == \

      "quicken install x86\n"

if __name__ == '__main__':

  pytest.main()

        shift

        ;;

      --compiler-filter)

        # ignore any '--compiler-filter xyz' settings.

        # FIXME: app_startup_runner.py should not be passing this flag.

        compiler_filter="$2"

        shift

        ;;

      *)

fi

verbose_print "Package was in path '$package_path'"

keep_application_trace_file=n

application_trace_file_path="$package_path/TraceFile.pb"

trace_file_directory="$package_path"

  fi

}

configure_compiler_filter() {

  local the_compiler_filter="$1"

  local the_package="$2"

  local the_activity="$3"

  if [[ -z $the_compiler_filter ]]; then

    verbose_print "No --compiler-filter specified, don't need to force it."

    return 0

  fi

  local current_compiler_filter_info="$("$DIR"/query_compiler_filter.py --package "$the_package")"

  local res=$?

  if [[ $res -ne 0 ]]; then

    return $res

  fi

  local current_compiler_filter

  local current_reason

  local current_isa

  read current_compiler_filter current_reason current_isa <<< "$current_compiler_filter_info"

  verbose_print "Compiler Filter="$current_compiler_filter "Reason="$current_reason "Isa="$current_isa

  # Don't trust reasons that aren't 'unknown' because that means we didn't manually force the compilation filter.

  # (e.g. if any automatic system-triggered compilations are not unknown).

  if [[ $current_reason != "unknown" ]] || [[ $current_compiler_filter != $the_compiler_filter ]]; then

    verbose_print "$DIR"/force_compiler_filter --compiler-filter "$the_compiler_filter" --package "$the_package" --activity "$the_activity"

    "$DIR"/force_compiler_filter --compiler-filter "$the_compiler_filter" --package "$the_package" --activity "$the_activity"

    res=$?

  else

    verbose_print "Queried compiler-filter matched requested compiler-filter, skip forcing."

    res=0

  fi

  return $res

}

# Ensure the APK is currently compiled with whatever we passed in via --compiler-filter.

# No-op if this option was not passed in.

configure_compiler_filter "$compiler_filter" "$package" "$activity" || exit 1

# TODO: This loop logic could probably be moved into app_startup_runner.py

for ((i=0;i<count;++i)) do

  verbose_print "=========================================="