Merge "Fix Google pylint warnings."

#!/usr/bin/env python

#!/usr/bin/python

# This file uses the following encoding: utf-8

"""Grep warnings messages and output HTML tables or warning counts in CSV.

Default is to output warnings in HTML tables grouped by warning severity.

Use option --byproject to output tables grouped by source file projects.

Use option --gencsv to output warning counts in CSV format.

"""

import argparse

import re

parser = argparse.ArgumentParser(description='Convert a build log into HTML')

parser.add_argument('--gencsv',

                    help='Generate a CSV file with number of various warnings',

                    action="store_true",

                    action='store_true',

                    default=False)

parser.add_argument('--byproject',

                    help='Separate warnings in HTML output by project names',

                    action="store_true",

                    action='store_true',

                    default=False)

parser.add_argument('--url',

                    help='Root URL of an Android source code tree prefixed '

                    help='Path to build.log file')

args = parser.parse_args()

# if you add another level, don't forget to give it a color below

class severity:

class severity:  # pylint:disable=invalid-name,old-style-class

  """Severity levels and attributes."""

  UNKNOWN = 0

  FIXMENOW = 1

  HIGH = 2

  HARMLESS = 6

  SKIP = 7

  attributes = [

      # pylint:disable=bad-whitespace

      ['lightblue', 'Unknown',   'Unknown warnings'],

      ['fuchsia',   'FixNow',    'Critical warnings, fix me now'],

      ['red',       'High',      'High severity warnings'],

      ['grey',      'Unhandled', 'Unhandled warnings']

  ]

  color = [a[0] for a in attributes]

    columnheader = [a[1] for a in attributes]

  column_headers = [a[1] for a in attributes]

  header = [a[2] for a in attributes]

  # order to dump by severity

  kinds = [FIXMENOW, HIGH, MEDIUM, LOW, TIDY, HARMLESS, UNKNOWN, SKIP]

warnpatterns = [

warn_patterns = [

    # TODO(chh): fix pylint space and indentation warnings

    # pylint:disable=bad-whitespace,bad-continuation,

    # pylint:disable=line-too-long,g-inconsistent-quotes

    { 'category':'make',    'severity':severity.MEDIUM,

        'description':'make: overriding commands/ignoring old commands',

        'patterns':[r".*: warning: overriding commands for target .+",

    { 'category':'C/C++',   'severity':severity.MEDIUM, 'option':'-Wmissing-noreturn',

        'description':'Missing noreturn',

        'patterns':[r".*: warning: function '.*' could be declared with attribute 'noreturn'"] },

    # pylint:disable=anomalous-backslash-in-string

    # TODO(chh): fix the backslash pylint warning.

    { 'category':'gcc',     'severity':severity.MEDIUM,

        'description':'Invalid option for C file',

        'patterns':[r".*: warning: command line option "".+"" is valid for C\+\+\/ObjC\+\+ but not for C"] },

        'patterns':[r".*: warning: .+"] },

]

for w in warnpatterns:

    w['members'] = []

    if 'option' not in w:

        w['option'] = ''

# A list of [project_name, file_path_pattern].

# project_name should not contain comma, to be used in CSV output.

projectlist = [

project_list = [

    # pylint:disable=bad-whitespace,g-inconsistent-quotes,line-too-long

    ['art',                 r"(^|.*/)art/.*: warning:"],

    ['bionic',              r"(^|.*/)bionic/.*: warning:"],

    ['bootable',            r"(^|.*/)bootable/.*: warning:"],

    ['other',   r".*: warning:"],

]

projectpatterns = []

for p in projectlist:

    projectpatterns.append({'description':p[0], 'members':[], 'pattern':re.compile(p[1])})

project_patterns = []

project_names = []

projectnames = [p[0] for p in projectlist]

def initialize_arrays():

  """Complete global arrays before they are used."""

  global project_names

  project_names = [p[0] for p in project_list]

  for p in project_list:

    project_patterns.append({'description': p[0],

                             'members': [],

                             'pattern': re.compile(p[1])})

  # Each warning pattern has 3 dictionaries:

  # (1) 'projects' maps a project name to number of warnings in that project.

# (2) 'projectanchor' maps a project name to its anchor number for HTML.

# (3) 'projectwarning' maps a project name to a list of warning of that project.

for w in warnpatterns:

  # (2) 'project_anchor' maps a project name to its anchor number for HTML.

  # (3) 'project_warnings' maps a project name to a list of warning messages.

  for w in warn_patterns:

    w['members'] = []

    if 'option' not in w:

      w['option'] = ''

    w['projects'] = {}

    w['projectanchor'] = {}

    w['projectwarning'] = {}

    w['project_anchor'] = {}

    w['project_warnings'] = {}

platformversion = 'unknown'

targetproduct = 'unknown'

targetvariant = 'unknown'

initialize_arrays()

platform_version = 'unknown'

target_product = 'unknown'

target_variant = 'unknown'

##### Data and functions to dump html file. ##################################

def output(text):

  print text,

def htmlbig(param):

def html_big(param):

  return '<font size="+2">' + param + '</font>'

def dumphtmlprologue(title):

def dump_html_prologue(title):

  output('<html>\n<head>\n')

  output('<title>' + title + '</title>\n')

  output(html_script_style)

  output('</head>\n<body>\n')

    output(htmlbig(title))

  output(html_big(title))

  output('<p>\n')

def dumphtmlepilogue():

def dump_html_epilogue():

  output('</body>\n</head>\n</html>\n')

def tablerow(text):

def table_row(text):

  global cur_row_class

  cur_row_class = 1 - cur_row_class

  # remove last '\n'

  t = text[:-1] if text[-1] == '\n' else text

  output('<tr><td class="c' + str(cur_row_class) + '">' + t + '</td></tr>\n')

def sortwarnings():

    for i in warnpatterns:

def sort_warnings():

  for i in warn_patterns:

    i['members'] = sorted(set(i['members']))

# dump a table of warnings per project and severity

def dumpstatsbyproject():

    projects = set(projectnames)

    severities = set(severity.kinds)

def dump_stats_by_project():

  """Dump a table of warnings per project and severity."""

  # warnings[p][s] is number of warnings in project p of severity s.

    warnings = {p:{s:0 for s in severity.kinds} for p in projectnames}

    for i in warnpatterns:

  warnings = {p: {s: 0 for s in severity.kinds} for p in project_names}

  for i in warn_patterns:

    s = i['severity']

    for p in i['projects']:

      warnings[p][s] += i['projects'][p]

    # totalbyproject[p] is number of warnings in project p.

    totalbyproject = {p:sum(warnings[p][s] for s in severity.kinds)

                      for p in projectnames}

  # total_by_project[p] is number of warnings in project p.

  total_by_project = {p: sum(warnings[p][s] for s in severity.kinds)

                      for p in project_names}

    # totalbyseverity[s] is number of warnings of severity s.

    totalbyseverity = {s:sum(warnings[p][s] for p in projectnames)

  # total_by_severity[s] is number of warnings of severity s.

  total_by_severity = {s: sum(warnings[p][s] for p in project_names)

                       for s in severity.kinds}

  # emit table header

  output('<blockquote><table border=1>\n<tr><th>Project</th>\n')

  for s in severity.kinds:

        if totalbyseverity[s]:

    if total_by_severity[s]:

      output('<th width="8%"><span style="background-color:{}">{}</span></th>'.

                   format(severity.color[s], severity.columnheader[s]))

             format(severity.color[s], severity.column_headers[s]))

  output('<th>TOTAL</th></tr>\n')

  # emit a row of warning counts per project, skip no-warning projects

    totalallprojects = 0

    for p in projectnames:

        if totalbyproject[p]:

  total_all_projects = 0

  for p in project_names:

    if total_by_project[p]:

      output('<tr><td align="left">{}</td>'.format(p))

      for s in severity.kinds:

                if totalbyseverity[s]:

        if total_by_severity[s]:

          output('<td align="right">{}</td>'.format(warnings[p][s]))

            output('<td align="right">{}</td>'.format(totalbyproject[p]))

            totalallprojects += totalbyproject[p]

      output('<td align="right">{}</td>'.format(total_by_project[p]))

      total_all_projects += total_by_project[p]

      output('</tr>\n')

  # emit a row of warning counts per severity

    totalallseverities = 0

  total_all_severities = 0

  output('<tr><td align="right">TOTAL</td>')

  for s in severity.kinds:

        if totalbyseverity[s]:

            output('<td align="right">{}</td>'.format(totalbyseverity[s]))

            totalallseverities += totalbyseverity[s]

    output('<td align="right">{}</td></tr>\n'.format(totalallprojects))

    if total_by_severity[s]:

      output('<td align="right">{}</td>'.format(total_by_severity[s]))

      total_all_severities += total_by_severity[s]

  output('<td align="right">{}</td></tr>\n'.format(total_all_projects))

  # at the end of table, verify total counts

  output('</table></blockquote><br>\n')

    if totalallprojects != totalallseverities:

        output('<h3>ERROR: Sum of warnings by project ' +

  if total_all_projects != total_all_severities:

    output('<h3>ERROR: Sum of warnings by project '

           '!= Sum of warnings by severity.</h3>\n')

# dump some stats about total number of warnings and such

def dumpstats():

def dump_stats():

  """Dump some stats about total number of warnings and such."""

  known = 0

  skipped = 0

  unknown = 0

    sortwarnings()

    for i in warnpatterns:

  sort_warnings()

  for i in warn_patterns:

    if i['severity'] == severity.UNKNOWN:

      unknown += len(i['members'])

    elif i['severity'] == severity.SKIP:

    output('(low count may indicate incremental build)')

  output('<br><br>\n')

def emitbuttons():

    output('<button class="button" onclick="expand_collapse(1);">' +

           'Expand all warnings</button>\n' +

           '<button class="button" onclick="expand_collapse(0);">' +

def emit_buttons():

  output('<button class="button" onclick="expand_collapse(1);">'

         'Expand all warnings</button>\n'

         '<button class="button" onclick="expand_collapse(0);">'

         'Collapse all warnings</button><br>\n')

# dump everything for a given severity

def dumpseverity(sev):

def dump_severity(sev):

  """Dump everything for a given severity."""

  global anchor

  total = 0

    for i in warnpatterns:

  for i in warn_patterns:

    if i['severity'] == sev:

            total = total + len(i['members'])

    output('\n<br><span style="background-color:' + severity.color[sev] + '"><b>' +

           severity.header[sev] + ': ' + str(total) + '</b></span>\n')

      total += len(i['members'])

  output('\n<br><span style="background-color:' + severity.color[sev] +

         '"><b>' + severity.header[sev] + ': ' + str(total) + '</b></span>\n')

  output('<blockquote>\n')

    for i in warnpatterns:

        if i['severity'] == sev and len(i['members']) > 0:

  for i in warn_patterns:

    if i['severity'] == sev and i['members']:

      anchor += 1

      i['anchor'] = str(anchor)

      if args.byproject:

                dumpcategorybyproject(sev, i)

        dump_category_by_project(sev, i)

      else:

                dumpcategory(sev, i)

        dump_category(sev, i)

  output('</blockquote>\n')

# emit all skipped project anchors for expand_collapse.

def dumpskippedanchors():

def dump_skipped_anchors():

  """emit all skipped project anchors for expand_collapse."""

  output('<div style="display:none;">\n')  # hide these fake elements

    for i in warnpatterns:

        if i['severity'] == severity.SKIP and len(i['members']) > 0:

            projects = i['projectwarning'].keys()

  for i in warn_patterns:

    if i['severity'] == severity.SKIP and i['members']:

      projects = i['project_warnings'].keys()

      for p in projects:

                output('<div id="' + i['projectanchor'][p] + '"></div>' +

                       '<div id="' + i['projectanchor'][p] + '_mark"></div>\n')

        output('<div id="' + i['project_anchor'][p] + '"></div>' +

               '<div id="' + i['project_anchor'][p] + '_mark"></div>\n')

  output('</div>\n')

def allpatterns(cat):

def all_patterns(cat):

  pats = ''

  for i in cat['patterns']:

    pats += i

    pats += ' / '

  return pats

def descriptionfor(cat):

    if cat['description'] != '':

def description_for(cat):

  if cat['description']:

    return cat['description']

    return allpatterns(cat)

  return all_patterns(cat)

# show which warnings no longer occur

def dumpfixed():

def dump_fixed():

  """Show which warnings no longer occur."""

  global anchor

  anchor += 1

  mark = str(anchor) + '_mark'

    output('\n<br><p style="background-color:lightblue"><b>' +

           '<button id="' + mark + '" ' +

           'class="bt" onclick="expand(' + str(anchor) + ');">' +

           '&#x2295</button> Fixed warnings. ' +

           'No more occurences. Please consider turning these into ' +

           'errors if possible, before they are reintroduced in to the build' +

  output('\n<br><p style="background-color:lightblue"><b>'

         '<button id="' + mark + '" '

         'class="bt" onclick="expand(' + str(anchor) + ');">'

         '&#x2295</button> Fixed warnings. '

         'No more occurrences. Please consider turning these into '

         'errors if possible, before they are reintroduced in to the build'

         ':</b></p>\n')

  output('<blockquote>\n')

  fixed_patterns = []

    for i in warnpatterns:

        if len(i['members']) == 0:

  for i in warn_patterns:

    if not i['members']:

      fixed_patterns.append(i['description'] + ' (' +

                                  allpatterns(i) + ')')

                            all_patterns(i) + ')')

    if i['option']:

      fixed_patterns.append(' ' + i['option'])

  fixed_patterns.sort()

  output('<div id="' + str(anchor) + '" style="display:none;"><table>\n')

  for i in fixed_patterns:

        tablerow(i)

    table_row(i)

  output('</table></div>\n')

  output('</blockquote>\n')

def warningwithurl(line):

def warning_with_url(line):

  """Returns a warning message line with HTML link to given args.url."""

  if not args.url:

    return line

  m = re.search(r'^([^ :]+):(\d+):(.+)', line, re.M|re.I)

  if not m:

    return line

    filepath = m.group(1)

    linenumber = m.group(2)

  file_path = m.group(1)

  line_number = m.group(2)

  warning = m.group(3)

  prefix = '<a href="' + args.url + '/' + file_path

  if args.separator:

        return '<a href="' + args.url + '/' + filepath + args.separator + linenumber + '">' + filepath + ':' + linenumber + '</a>:' + warning

    return (prefix + args.separator + line_number + '">' + file_path +

            ':' + line_number + '</a>:' + warning)

  else:

        return '<a href="' + args.url + '/' + filepath + '">' + filepath + '</a>:' + linenumber + ':' + warning

    return prefix + '">' + file_path + '</a>:' + line_number + ':' + warning

def dumpgroup(sev, anchor, description, warnings):

    mark = anchor + '_mark'

def dump_group(sev, anchor_str, description, warnings):

  """Dump warnings of given severity, anchor_str, and description."""

  mark = anchor_str + '_mark'

  output('\n<table class="t1">\n')

  output('<tr bgcolor="' + severity.color[sev] + '">' +

         '<td><button class="bt" id="' + mark +

           '" onclick="expand(\'' + anchor + '\');">' +

         '" onclick="expand(\'' + anchor_str + '\');">' +

         '&#x2295</button> ' + description + '</td></tr>\n')

  output('</table>\n')

    output('<div id="' + anchor + '" style="display:none;">')

  output('<div id="' + anchor_str + '" style="display:none;">')

  output('<table class="t1">\n')

  for i in warnings:

        tablerow(warningwithurl(i))

    table_row(warning_with_url(i))

  output('</table></div>\n')

# dump warnings in a category

def dumpcategory(sev, cat):

    description = descriptionfor(cat) + ' (' + str(len(cat['members'])) + ')'

    dumpgroup(sev, cat['anchor'], description, cat['members'])

# similar to dumpcategory but output one table per project.

def dumpcategorybyproject(sev, cat):

    warning = descriptionfor(cat)

    projects = cat['projectwarning'].keys()

def dump_category(sev, cat):

  """Dump warnings in a category."""

  description = description_for(cat) + ' (' + str(len(cat['members'])) + ')'

  dump_group(sev, cat['anchor'], description, cat['members'])

def dump_category_by_project(sev, cat):

  """Similar to dump_category but output one table per project."""

  warning = description_for(cat)

  projects = cat['project_warnings'].keys()

  projects.sort()

  for p in projects:

        anchor = cat['projectanchor'][p]

        projectwarnings = cat['projectwarning'][p]

        description = '{}, in {} ({})'.format(warning, p, len(projectwarnings))

        dumpgroup(sev, anchor, description, projectwarnings)

    anchor_str = cat['project_anchor'][p]

    project_warnings = cat['project_warnings'][p]

    description = '{}, in {} ({})'.format(warning, p, len(project_warnings))

    dump_group(sev, anchor_str, description, project_warnings)

def findproject(line):

    for p in projectpatterns:

def find_project(line):

  for p in project_patterns:

    if p['pattern'].match(line):

      return p['description']

  return '???'

def classifywarning(line):

def classify_warning(line):

  global anchor

    for i in warnpatterns:

        for cpat in i['compiledpatterns']:

  for i in warn_patterns:

    for cpat in i['compiled_patterns']:

      if cpat.match(line):

        i['members'].append(line)

                pname = findproject(line)

        pname = find_project(line)

        # Count warnings by project.

        if pname in i['projects']:

          i['projects'][pname] += 1

          i['projects'][pname] = 1

        # Collect warnings by project.

        if args.byproject:

                    if pname in i['projectwarning']:

                        i['projectwarning'][pname].append(line)

          if pname in i['project_warnings']:

            i['project_warnings'][pname].append(line)

          else:

                        i['projectwarning'][pname] = [line]

                    if pname not in i['projectanchor']:

            i['project_warnings'][pname] = [line]

          if pname not in i['project_anchor']:

            anchor += 1

                        i['projectanchor'][pname] = str(anchor)

            i['project_anchor'][pname] = str(anchor)

        return

      else:

        # If we end up here, there was a problem parsing the log

        # 2 or more concurrent compiles

        pass

# precompiling every pattern speeds up parsing by about 30x

def compilepatterns():

    for i in warnpatterns:

        i['compiledpatterns'] = []

def compile_patterns():

  """Precompiling every pattern speeds up parsing by about 30x."""

  for i in warn_patterns:

    i['compiled_patterns'] = []

    for pat in i['patterns']:

            i['compiledpatterns'].append(re.compile(pat))

      i['compiled_patterns'].append(re.compile(pat))

def parseinputfile():

    global platformversion

    global targetproduct

    global targetvariant

def parse_input_file():

  """Parse input file, match warning lines."""

  global platform_version

  global target_product

  global target_variant

  infile = open(args.buildlog, 'r')

    linecounter = 0

  line_counter = 0

    warningpattern = re.compile('.* warning:.*')

    compilepatterns()

  warning_pattern = re.compile('.* warning:.*')

  compile_patterns()