Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 992d0860 authored by Chih-hung Hsieh's avatar Chih-hung Hsieh Committed by Automerger Merge Worker
Browse files

Merge "Handle more mixed warning lines from RBE" am: b28e4c22 am: c7638c66

parents 9fe03ee3 c7638c66
Loading
Loading
Loading
Loading
+71 −25
Original line number Original line Diff line number Diff line
@@ -228,6 +228,14 @@ def find_warn_py_and_android_root(path):
  return ''
  return ''




def find_android_root_top_dirs(root_dir):
  """Return a list of directories under the root_dir, if it exists."""
  if not os.path.isdir(root_dir):
    return None
  return list(filter(lambda d: os.path.isdir(root_dir + '/' + d),
                     os.listdir(root_dir)))


def find_android_root(buildlog):
def find_android_root(buildlog):
  """Guess android source root from common prefix of file paths."""
  """Guess android source root from common prefix of file paths."""
  # Use the longest common prefix of the absolute file paths
  # Use the longest common prefix of the absolute file paths
@@ -239,8 +247,8 @@ def find_android_root(buildlog):
    # We want to find android_root of a local build machine.
    # We want to find android_root of a local build machine.
    # Do not use RBE warning lines, which has '/b/f/w/' path prefix.
    # Do not use RBE warning lines, which has '/b/f/w/' path prefix.
    # Do not use /tmp/ file warnings.
    # Do not use /tmp/ file warnings.
    if warning_pattern.match(line) and (
    if ('/b/f/w' not in line and not line.startswith('/tmp/') and
        '/b/f/w' not in line and not line.startswith('/tmp/')):
        warning_pattern.match(line)):
      warning_lines.append(line)
      warning_lines.append(line)
      count += 1
      count += 1
      if count > 9999:
      if count > 9999:
@@ -251,15 +259,23 @@ def find_android_root(buildlog):
        path = os.path.normpath(re.sub(':.*$', '', line))
        path = os.path.normpath(re.sub(':.*$', '', line))
        android_root = find_warn_py_and_android_root(path)
        android_root = find_warn_py_and_android_root(path)
        if android_root:
        if android_root:
          return android_root
          return android_root, find_android_root_top_dirs(android_root)
  # Do not use common prefix of a small number of paths.
  # Do not use common prefix of a small number of paths.
  android_root = ''
  if count > 10:
  if count > 10:
    # pytype: disable=wrong-arg-types
    # pytype: disable=wrong-arg-types
    root_path = os.path.commonprefix(warning_lines)
    root_path = os.path.commonprefix(warning_lines)
    # pytype: enable=wrong-arg-types
    # pytype: enable=wrong-arg-types
    if len(root_path) > 2 and root_path[len(root_path) - 1] == '/':
    if len(root_path) > 2 and root_path[len(root_path) - 1] == '/':
      return root_path[:-1]
      android_root = root_path[:-1]
  return ''
  if android_root and os.path.isdir(android_root):
    return android_root, find_android_root_top_dirs(android_root)
  # When the build.log file is moved to a different machine where
  # android_root is not found, use the location of this script
  # to find the android source tree root and its sub directories.
  # This __file__ is /..../build/make/tools/warn/warn_common.py
  script_root = __file__.replace('/build/make/tools/warn/warn_common.py', '')
  return android_root, find_android_root_top_dirs(script_root)




def remove_android_root_prefix(path, android_root):
def remove_android_root_prefix(path, android_root):
@@ -310,8 +326,6 @@ def parse_input_file_chrome(infile, flags):
  warning_pattern = re.compile(chrome_warning_pattern)
  warning_pattern = re.compile(chrome_warning_pattern)


  # Collect all unique warning lines
  # Collect all unique warning lines
  # Remove the duplicated warnings save ~8% of time when parsing
  # one typical build log than before
  unique_warnings = dict()
  unique_warnings = dict()
  for line in infile:
  for line in infile:
    if warning_pattern.match(line):
    if warning_pattern.match(line):
@@ -354,7 +368,7 @@ def parse_input_file_android(infile, flags):
  target_variant = 'unknown'
  target_variant = 'unknown'
  build_id = 'unknown'
  build_id = 'unknown'
  use_rbe = False
  use_rbe = False
  android_root = find_android_root(infile)
  android_root, root_top_dirs = find_android_root(infile)
  infile.seek(0)
  infile.seek(0)


  # rustc warning messages have two lines that should be combined:
  # rustc warning messages have two lines that should be combined:
@@ -367,24 +381,39 @@ def parse_input_file_android(infile, flags):
  # C/C++ compiler warning messages have line and column numbers:
  # C/C++ compiler warning messages have line and column numbers:
  #     some/path/file.c:line_number:column_number: warning: description
  #     some/path/file.c:line_number:column_number: warning: description
  warning_pattern = re.compile('(^[^ ]*/[^ ]*: warning: .*)|(^warning: .*)')
  warning_pattern = re.compile('(^[^ ]*/[^ ]*: warning: .*)|(^warning: .*)')
  warning_without_file = re.compile('^warning: .*')
  rustc_file_position = re.compile('^[ ]+--> [^ ]*/[^ ]*:[0-9]+:[0-9]+')
  rustc_file_position = re.compile('^[ ]+--> [^ ]*/[^ ]*:[0-9]+:[0-9]+')


  # If RBE was used, try to reclaim some warning lines mixed with some
  # If RBE was used, try to reclaim some warning lines (from stdout)
  # leading chars from other concurrent job's stderr output .
  # that contain leading characters from stderr.
  # The leading characters can be any character, including digits and spaces.
  # The leading characters can be any character, including digits and spaces.
  # It's impossible to correctly identify the starting point of the source

  # file path without the file directory name knowledge.
  # If a warning line's source file path contains the special RBE prefix
  # Here we can only be sure to recover lines containing "/b/f/w/".
  # /b/f/w/, we can remove all leading chars up to and including the "/b/f/w/".
  rbe_warning_pattern = re.compile('.*/b/f/w/[^ ]*: warning: .*')
  bfw_warning_pattern = re.compile('.*/b/f/w/([^ ]*: warning: .*)')

  # When android_root is known and available, we find its top directories
  # and remove all leading chars before a top directory name.
  # We assume that the leading chars from stderr do not contain "/".
  # For example,
  #   10external/...
  #   12 warningsexternal/...
  #   413 warningexternal/...
  #   5 warnings generatedexternal/...
  #   Suppressed 1000 warnings (packages/modules/...
  if root_top_dirs:
    extra_warning_pattern = re.compile(
        '^.[^/]*((' + '|'.join(root_top_dirs) +
        ')/[^ ]*: warning: .*)')
  else:
    extra_warning_pattern = re.compile('^[^/]* ([^ /]*/[^ ]*: warning: .*)')


  # Collect all unique warning lines
  # Collect all unique warning lines
  # Remove the duplicated warnings save ~8% of time when parsing
  # one typical build log than before
  unique_warnings = dict()
  unique_warnings = dict()
  checked_warning_lines = dict()
  line_counter = 0
  line_counter = 0
  prev_warning = ''
  prev_warning = ''
  for line in infile:
  for line in infile:
    line_counter += 1
    if prev_warning:
    if prev_warning:
      if rustc_file_position.match(line):
      if rustc_file_position.match(line):
        # must be a rustc warning, combine 2 lines into one warning
        # must be a rustc warning, combine 2 lines into one warning
@@ -399,14 +428,32 @@ def parse_input_file_android(infile, flags):
          prev_warning, flags, android_root, unique_warnings)
          prev_warning, flags, android_root, unique_warnings)
      prev_warning = ''
      prev_warning = ''


    if use_rbe and rbe_warning_pattern.match(line):
    # re.match is slow, with several warning line patterns and
      cleaned_up_line = re.sub('.*/b/f/w/', '', line)
    # long input lines like "TIMEOUT: ...".
      unique_warnings = add_normalized_line_to_warnings(
    # We save significant time by skipping non-warning lines.
          cleaned_up_line, flags, android_root, unique_warnings)
    # But do not skip the first 100 lines, because we want to
    # catch build variables.
    if line_counter > 100 and line.find('warning: ') < 0:
      continue
      continue


    # A large clean build output can contain up to 90% of duplicated
    # "warning:" lines. If we can skip them quickly, we can
    # speed up this for-loop 3X to 5X.
    if line in checked_warning_lines:
      continue
    checked_warning_lines[line] = True

    # Clean up extra prefix if RBE is used.
    if use_rbe:
      if '/b/f/w/' in line:
        result = bfw_warning_pattern.search(line)
      else:
        result = extra_warning_pattern.search(line)
      if result is not None:
        line = result.group(1)

    if warning_pattern.match(line):
    if warning_pattern.match(line):
      if warning_without_file.match(line):
      if line.startswith('warning: '):
        # save this line and combine it with the next line
        # save this line and combine it with the next line
        prev_warning = line
        prev_warning = line
      else:
      else:
@@ -416,7 +463,6 @@ def parse_input_file_android(infile, flags):


    if line_counter < 100:
    if line_counter < 100:
      # save a little bit of time by only doing this for the first few lines
      # save a little bit of time by only doing this for the first few lines
      line_counter += 1
      result = re.search('(?<=^PLATFORM_VERSION=).*', line)
      result = re.search('(?<=^PLATFORM_VERSION=).*', line)
      if result is not None:
      if result is not None:
        platform_version = result.group(0)
        platform_version = result.group(0)