Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 6c0fdbb8 authored by Chih-Hung Hsieh's avatar Chih-Hung Hsieh
Browse files

Fix Google pylint warnings. 

* Rename variables and functions to follow Google coding style.
* Fix line indentations to follow Google coding style.
* Add module and function docstrings.
* Change shebang line to avoid pylint warning.
* Suppress some pylint warnings to avoid too many changes at once.
* Fix typos.

Test: run warn.py --byproject build.log
Change-Id: I569961981b562e35cb5609a96cd5ebb40a80829c
parent 19f24492

tools/warn.py

+450 −397
Original line number Diff line number Diff line 
#!/usr/bin/env python

#!/usr/bin/python

# This file uses the following encoding: utf-8



"""Grep warnings messages and output HTML tables or warning counts in CSV.



Default is to output warnings in HTML tables grouped by warning severity.

Use option --byproject to output tables grouped by source file projects.

Use option --gencsv to output warning counts in CSV format.

"""



import argparse

import re



parser = argparse.ArgumentParser(description='Convert a build log into HTML')

parser.add_argument('--gencsv',

                    help='Generate a CSV file with number of various warnings',

                    action="store_true",

                    action='store_true',

                    default=False)

parser.add_argument('--byproject',

                    help='Separate warnings in HTML output by project names',

                    action="store_true",

                    action='store_true',

                    default=False)

parser.add_argument('--url',

                    help='Root URL of an Android source code tree prefixed '
@@ -23,8 +30,10 @@ parser.add_argument(dest='buildlog', metavar='build.log', 
                    help='Path to build.log file')

args = parser.parse_args()





# if you add another level, don't forget to give it a color below

class severity:

class severity:  # pylint:disable=invalid-name,old-style-class

  """Severity levels and attributes."""

  UNKNOWN = 0

  FIXMENOW = 1

  HIGH = 2
@@ -34,6 +43,7 @@ class severity: 
  HARMLESS = 6

  SKIP = 7

  attributes = [

      # pylint:disable=bad-whitespace

      ['lightblue', 'Unknown',   'Unknown warnings'],

      ['fuchsia',   'FixNow',    'Critical warnings, fix me now'],

      ['red',       'High',      'High severity warnings'],
@@ -44,12 +54,15 @@ class severity: 
      ['grey',      'Unhandled', 'Unhandled warnings']

  ]

  color = [a[0] for a in attributes]

    columnheader = [a[1] for a in attributes]

  column_headers = [a[1] for a in attributes]

  header = [a[2] for a in attributes]

  # order to dump by severity

  kinds = [FIXMENOW, HIGH, MEDIUM, LOW, TIDY, HARMLESS, UNKNOWN, SKIP]



warnpatterns = [

warn_patterns = [

    # TODO(chh): fix pylint space and indentation warnings

    # pylint:disable=bad-whitespace,bad-continuation,

    # pylint:disable=line-too-long,g-inconsistent-quotes

    { 'category':'make',    'severity':severity.MEDIUM,

        'description':'make: overriding commands/ignoring old commands',

        'patterns':[r".*: warning: overriding commands for target .+",
@@ -1155,6 +1168,8 @@ warnpatterns = [ 
    { 'category':'C/C++',   'severity':severity.MEDIUM, 'option':'-Wmissing-noreturn',

        'description':'Missing noreturn',

        'patterns':[r".*: warning: function '.*' could be declared with attribute 'noreturn'"] },

    # pylint:disable=anomalous-backslash-in-string

    # TODO(chh): fix the backslash pylint warning.

    { 'category':'gcc',     'severity':severity.MEDIUM,

        'description':'Invalid option for C file',

        'patterns':[r".*: warning: command line option "".+"" is valid for C\+\+\/ObjC\+\+ but not for C"] },
@@ -1571,14 +1586,11 @@ warnpatterns = [ 
        'patterns':[r".*: warning: .+"] },

]



for w in warnpatterns:

    w['members'] = []

    if 'option' not in w:

        w['option'] = ''



# A list of [project_name, file_path_pattern].

# project_name should not contain comma, to be used in CSV output.

projectlist = [

project_list = [

    # pylint:disable=bad-whitespace,g-inconsistent-quotes,line-too-long

    ['art',                 r"(^|.*/)art/.*: warning:"],

    ['bionic',              r"(^|.*/)bionic/.*: warning:"],

    ['bootable',            r"(^|.*/)bootable/.*: warning:"],
@@ -1613,24 +1625,37 @@ projectlist = [ 
    ['other',   r".*: warning:"],

]



projectpatterns = []

for p in projectlist:

    projectpatterns.append({'description':p[0], 'members':[], 'pattern':re.compile(p[1])})

project_patterns = []

project_names = []



projectnames = [p[0] for p in projectlist]



def initialize_arrays():

  """Complete global arrays before they are used."""

  global project_names

  project_names = [p[0] for p in project_list]

  for p in project_list:

    project_patterns.append({'description': p[0],

                             'members': [],

                             'pattern': re.compile(p[1])})

  # Each warning pattern has 3 dictionaries:

  # (1) 'projects' maps a project name to number of warnings in that project.

# (2) 'projectanchor' maps a project name to its anchor number for HTML.

# (3) 'projectwarning' maps a project name to a list of warning of that project.

for w in warnpatterns:

  # (2) 'project_anchor' maps a project name to its anchor number for HTML.

  # (3) 'project_warnings' maps a project name to a list of warning messages.

  for w in warn_patterns:

    w['members'] = []

    if 'option' not in w:

      w['option'] = ''

    w['projects'] = {}

    w['projectanchor'] = {}

    w['projectwarning'] = {}

    w['project_anchor'] = {}

    w['project_warnings'] = {}





platformversion = 'unknown'

targetproduct = 'unknown'

targetvariant = 'unknown'

initialize_arrays()





platform_version = 'unknown'

target_product = 'unknown'

target_variant = 'unknown'





##### Data and functions to dump html file. ##################################
@@ -1676,93 +1701,97 @@ html_script_style = """\ 
def output(text):

  print text,



def htmlbig(param):



def html_big(param):

  return '<font size="+2">' + param + '</font>'



def dumphtmlprologue(title):



def dump_html_prologue(title):

  output('<html>\n<head>\n')

  output('<title>' + title + '</title>\n')

  output(html_script_style)

  output('</head>\n<body>\n')

    output(htmlbig(title))

  output(html_big(title))

  output('<p>\n')



def dumphtmlepilogue():



def dump_html_epilogue():

  output('</body>\n</head>\n</html>\n')



def tablerow(text):



def table_row(text):

  global cur_row_class

  cur_row_class = 1 - cur_row_class

  # remove last '\n'

  t = text[:-1] if text[-1] == '\n' else text

  output('<tr><td class="c' + str(cur_row_class) + '">' + t + '</td></tr>\n')



def sortwarnings():

    for i in warnpatterns:



def sort_warnings():

  for i in warn_patterns:

    i['members'] = sorted(set(i['members']))



# dump a table of warnings per project and severity

def dumpstatsbyproject():

    projects = set(projectnames)

    severities = set(severity.kinds)



def dump_stats_by_project():

  """Dump a table of warnings per project and severity."""

  # warnings[p][s] is number of warnings in project p of severity s.

    warnings = {p:{s:0 for s in severity.kinds} for p in projectnames}

    for i in warnpatterns:

  warnings = {p: {s: 0 for s in severity.kinds} for p in project_names}

  for i in warn_patterns:

    s = i['severity']

    for p in i['projects']:

      warnings[p][s] += i['projects'][p]



    # totalbyproject[p] is number of warnings in project p.

    totalbyproject = {p:sum(warnings[p][s] for s in severity.kinds)

                      for p in projectnames}

  # total_by_project[p] is number of warnings in project p.

  total_by_project = {p: sum(warnings[p][s] for s in severity.kinds)

                      for p in project_names}



    # totalbyseverity[s] is number of warnings of severity s.

    totalbyseverity = {s:sum(warnings[p][s] for p in projectnames)

  # total_by_severity[s] is number of warnings of severity s.

  total_by_severity = {s: sum(warnings[p][s] for p in project_names)

                       for s in severity.kinds}



  # emit table header

  output('<blockquote><table border=1>\n<tr><th>Project</th>\n')

  for s in severity.kinds:

        if totalbyseverity[s]:

    if total_by_severity[s]:

      output('<th width="8%"><span style="background-color:{}">{}</span></th>'.

                   format(severity.color[s], severity.columnheader[s]))

             format(severity.color[s], severity.column_headers[s]))

  output('<th>TOTAL</th></tr>\n')



  # emit a row of warning counts per project, skip no-warning projects

    totalallprojects = 0

    for p in projectnames:

        if totalbyproject[p]:

  total_all_projects = 0

  for p in project_names:

    if total_by_project[p]:

      output('<tr><td align="left">{}</td>'.format(p))

      for s in severity.kinds:

                if totalbyseverity[s]:

        if total_by_severity[s]:

          output('<td align="right">{}</td>'.format(warnings[p][s]))

            output('<td align="right">{}</td>'.format(totalbyproject[p]))

            totalallprojects += totalbyproject[p]

      output('<td align="right">{}</td>'.format(total_by_project[p]))

      total_all_projects += total_by_project[p]

      output('</tr>\n')



  # emit a row of warning counts per severity

    totalallseverities = 0

  total_all_severities = 0

  output('<tr><td align="right">TOTAL</td>')

  for s in severity.kinds:

        if totalbyseverity[s]:

            output('<td align="right">{}</td>'.format(totalbyseverity[s]))

            totalallseverities += totalbyseverity[s]

    output('<td align="right">{}</td></tr>\n'.format(totalallprojects))

    if total_by_severity[s]:

      output('<td align="right">{}</td>'.format(total_by_severity[s]))

      total_all_severities += total_by_severity[s]

  output('<td align="right">{}</td></tr>\n'.format(total_all_projects))



  # at the end of table, verify total counts

  output('</table></blockquote><br>\n')

    if totalallprojects != totalallseverities:

        output('<h3>ERROR: Sum of warnings by project ' +

  if total_all_projects != total_all_severities:

    output('<h3>ERROR: Sum of warnings by project '

           '!= Sum of warnings by severity.</h3>\n')



# dump some stats about total number of warnings and such

def dumpstats():



def dump_stats():

  """Dump some stats about total number of warnings and such."""

  known = 0

  skipped = 0

  unknown = 0

    sortwarnings()

    for i in warnpatterns:

  sort_warnings()

  for i in warn_patterns:

    if i['severity'] == severity.UNKNOWN:

      unknown += len(i['members'])

    elif i['severity'] == severity.SKIP:
@@ -1778,140 +1807,155 @@ def dumpstats(): 
    output('(low count may indicate incremental build)')

  output('<br><br>\n')



def emitbuttons():

    output('<button class="button" onclick="expand_collapse(1);">' +

           'Expand all warnings</button>\n' +

           '<button class="button" onclick="expand_collapse(0);">' +



def emit_buttons():

  output('<button class="button" onclick="expand_collapse(1);">'

         'Expand all warnings</button>\n'

         '<button class="button" onclick="expand_collapse(0);">'

         'Collapse all warnings</button><br>\n')



# dump everything for a given severity

def dumpseverity(sev):



def dump_severity(sev):

  """Dump everything for a given severity."""

  global anchor

  total = 0

    for i in warnpatterns:

  for i in warn_patterns:

    if i['severity'] == sev:

            total = total + len(i['members'])

    output('\n<br><span style="background-color:' + severity.color[sev] + '"><b>' +

           severity.header[sev] + ': ' + str(total) + '</b></span>\n')

      total += len(i['members'])

  output('\n<br><span style="background-color:' + severity.color[sev] +

         '"><b>' + severity.header[sev] + ': ' + str(total) + '</b></span>\n')

  output('<blockquote>\n')

    for i in warnpatterns:

        if i['severity'] == sev and len(i['members']) > 0:

  for i in warn_patterns:

    if i['severity'] == sev and i['members']:

      anchor += 1

      i['anchor'] = str(anchor)

      if args.byproject:

                dumpcategorybyproject(sev, i)

        dump_category_by_project(sev, i)

      else:

                dumpcategory(sev, i)

        dump_category(sev, i)

  output('</blockquote>\n')



# emit all skipped project anchors for expand_collapse.

def dumpskippedanchors():



def dump_skipped_anchors():

  """emit all skipped project anchors for expand_collapse."""

  output('<div style="display:none;">\n')  # hide these fake elements

    for i in warnpatterns:

        if i['severity'] == severity.SKIP and len(i['members']) > 0:

            projects = i['projectwarning'].keys()

  for i in warn_patterns:

    if i['severity'] == severity.SKIP and i['members']:

      projects = i['project_warnings'].keys()

      for p in projects:

                output('<div id="' + i['projectanchor'][p] + '"></div>' +

                       '<div id="' + i['projectanchor'][p] + '_mark"></div>\n')

        output('<div id="' + i['project_anchor'][p] + '"></div>' +

               '<div id="' + i['project_anchor'][p] + '_mark"></div>\n')

  output('</div>\n')



def allpatterns(cat):



def all_patterns(cat):

  pats = ''

  for i in cat['patterns']:

    pats += i

    pats += ' / '

  return pats



def descriptionfor(cat):

    if cat['description'] != '':



def description_for(cat):

  if cat['description']:

    return cat['description']

    return allpatterns(cat)

  return all_patterns(cat)





# show which warnings no longer occur

def dumpfixed():

def dump_fixed():

  """Show which warnings no longer occur."""

  global anchor

  anchor += 1

  mark = str(anchor) + '_mark'

    output('\n<br><p style="background-color:lightblue"><b>' +

           '<button id="' + mark + '" ' +

           'class="bt" onclick="expand(' + str(anchor) + ');">' +

           '&#x2295</button> Fixed warnings. ' +

           'No more occurences. Please consider turning these into ' +

           'errors if possible, before they are reintroduced in to the build' +

  output('\n<br><p style="background-color:lightblue"><b>'

         '<button id="' + mark + '" '

         'class="bt" onclick="expand(' + str(anchor) + ');">'

         '&#x2295</button> Fixed warnings. '

         'No more occurrences. Please consider turning these into '

         'errors if possible, before they are reintroduced in to the build'

         ':</b></p>\n')

  output('<blockquote>\n')

  fixed_patterns = []

    for i in warnpatterns:

        if len(i['members']) == 0:

  for i in warn_patterns:

    if not i['members']:

      fixed_patterns.append(i['description'] + ' (' +

                                  allpatterns(i) + ')')

                            all_patterns(i) + ')')

    if i['option']:

      fixed_patterns.append(' ' + i['option'])

  fixed_patterns.sort()

  output('<div id="' + str(anchor) + '" style="display:none;"><table>\n')

  for i in fixed_patterns:

        tablerow(i)

    table_row(i)

  output('</table></div>\n')

  output('</blockquote>\n')



def warningwithurl(line):



def warning_with_url(line):

  """Returns a warning message line with HTML link to given args.url."""

  if not args.url:

    return line

  m = re.search(r'^([^ :]+):(\d+):(.+)', line, re.M|re.I)

  if not m:

    return line

    filepath = m.group(1)

    linenumber = m.group(2)

  file_path = m.group(1)

  line_number = m.group(2)

  warning = m.group(3)

  prefix = '<a href="' + args.url + '/' + file_path

  if args.separator:

        return '<a href="' + args.url + '/' + filepath + args.separator + linenumber + '">' + filepath + ':' + linenumber + '</a>:' + warning

    return (prefix + args.separator + line_number + '">' + file_path +

            ':' + line_number + '</a>:' + warning)

  else:

        return '<a href="' + args.url + '/' + filepath + '">' + filepath + '</a>:' + linenumber + ':' + warning

    return prefix + '">' + file_path + '</a>:' + line_number + ':' + warning



def dumpgroup(sev, anchor, description, warnings):

    mark = anchor + '_mark'



def dump_group(sev, anchor_str, description, warnings):

  """Dump warnings of given severity, anchor_str, and description."""

  mark = anchor_str + '_mark'

  output('\n<table class="t1">\n')

  output('<tr bgcolor="' + severity.color[sev] + '">' +

         '<td><button class="bt" id="' + mark +

           '" onclick="expand(\'' + anchor + '\');">' +

         '" onclick="expand(\'' + anchor_str + '\');">' +

         '&#x2295</button> ' + description + '</td></tr>\n')

  output('</table>\n')

    output('<div id="' + anchor + '" style="display:none;">')

  output('<div id="' + anchor_str + '" style="display:none;">')

  output('<table class="t1">\n')

  for i in warnings:

        tablerow(warningwithurl(i))

    table_row(warning_with_url(i))

  output('</table></div>\n')



# dump warnings in a category

def dumpcategory(sev, cat):

    description = descriptionfor(cat) + ' (' + str(len(cat['members'])) + ')'

    dumpgroup(sev, cat['anchor'], description, cat['members'])



# similar to dumpcategory but output one table per project.

def dumpcategorybyproject(sev, cat):

    warning = descriptionfor(cat)

    projects = cat['projectwarning'].keys()

def dump_category(sev, cat):

  """Dump warnings in a category."""

  description = description_for(cat) + ' (' + str(len(cat['members'])) + ')'

  dump_group(sev, cat['anchor'], description, cat['members'])





def dump_category_by_project(sev, cat):

  """Similar to dump_category but output one table per project."""

  warning = description_for(cat)

  projects = cat['project_warnings'].keys()

  projects.sort()

  for p in projects:

        anchor = cat['projectanchor'][p]

        projectwarnings = cat['projectwarning'][p]

        description = '{}, in {} ({})'.format(warning, p, len(projectwarnings))

        dumpgroup(sev, anchor, description, projectwarnings)

    anchor_str = cat['project_anchor'][p]

    project_warnings = cat['project_warnings'][p]

    description = '{}, in {} ({})'.format(warning, p, len(project_warnings))

    dump_group(sev, anchor_str, description, project_warnings)



def findproject(line):

    for p in projectpatterns:



def find_project(line):

  for p in project_patterns:

    if p['pattern'].match(line):

      return p['description']

  return '???'



def classifywarning(line):



def classify_warning(line):

  global anchor

    for i in warnpatterns:

        for cpat in i['compiledpatterns']:

  for i in warn_patterns:

    for cpat in i['compiled_patterns']:

      if cpat.match(line):

        i['members'].append(line)

                pname = findproject(line)

        pname = find_project(line)

        # Count warnings by project.

        if pname in i['projects']:

          i['projects'][pname] += 1
@@ -1919,13 +1963,13 @@ def classifywarning(line): 
          i['projects'][pname] = 1

        # Collect warnings by project.

        if args.byproject:

                    if pname in i['projectwarning']:

                        i['projectwarning'][pname].append(line)

          if pname in i['project_warnings']:

            i['project_warnings'][pname].append(line)

          else:

                        i['projectwarning'][pname] = [line]

                    if pname not in i['projectanchor']:

            i['project_warnings'][pname] = [line]

          if pname not in i['project_anchor']:

            anchor += 1

                        i['projectanchor'][pname] = str(anchor)

            i['project_anchor'][pname] = str(anchor)

        return

      else:

        # If we end up here, there was a problem parsing the log
@@ -1933,104 +1977,113 @@ def classifywarning(line): 
        # 2 or more concurrent compiles

        pass



# precompiling every pattern speeds up parsing by about 30x

def compilepatterns():

    for i in warnpatterns:

        i['compiledpatterns'] = []



def compile_patterns():

  """Precompiling every pattern speeds up parsing by about 30x."""

  for i in warn_patterns:

    i['compiled_patterns'] = []

    for pat in i['patterns']:

            i['compiledpatterns'].append(re.compile(pat))

      i['compiled_patterns'].append(re.compile(pat))





def parseinputfile():

    global platformversion

    global targetproduct

    global targetvariant

def parse_input_file():

  """Parse input file, match warning lines."""

  global platform_version

  global target_product

  global target_variant

  infile = open(args.buildlog, 'r')

    linecounter = 0

  line_counter = 0



    warningpattern = re.compile('.* warning:.*')

    compilepatterns()

  warning_pattern = re.compile('.* warning:.*')

  compile_patterns()



  # read the log file and classify all the warnings

    warninglines = set()

  warning_lines = set()

  for line in infile:

    # replace fancy quotes with plain ol' quotes

        line = line.replace("‘", "'");

        line = line.replace("’", "'");

        if warningpattern.match(line):

            if line not in warninglines:

                classifywarning(line)

                warninglines.add(line)

    line = line.replace('‘', "'")

    line = line.replace('’', "'")

    if warning_pattern.match(line):

      if line not in warning_lines:

        classify_warning(line)

        warning_lines.add(line)

    else:

      # save a little bit of time by only doing this for the first few lines

            if linecounter < 50:

                linecounter +=1

      if line_counter < 50:

        line_counter += 1

        m = re.search('(?<=^PLATFORM_VERSION=).*', line)

                if m != None:

                    platformversion = m.group(0)

        if m is not None:

          platform_version = m.group(0)

        m = re.search('(?<=^TARGET_PRODUCT=).*', line)

                if m != None:

                    targetproduct = m.group(0)

        if m is not None:

          target_product = m.group(0)

        m = re.search('(?<=^TARGET_BUILD_VARIANT=).*', line)

                if m != None:

                    targetvariant = m.group(0)





# dump the html output to stdout

def dumphtml():

    dumphtmlprologue('Warnings for ' + platformversion + ' - ' + targetproduct + ' - ' + targetvariant)

    dumpstats()

    dumpstatsbyproject()

    emitbuttons()

    # sort table based on number of members once dumpstats has deduplicated the

        if m is not None:

          target_variant = m.group(0)





def dump_html():

  """Dump the html output to stdout."""

  dump_html_prologue('Warnings for ' + platform_version + ' - ' +

                     target_product + ' - ' + target_variant)

  dump_stats()

  dump_stats_by_project()

  emit_buttons()

  # sort table based on number of members once dump_stats has de-duplicated the

  # members.

    warnpatterns.sort(reverse=True, key=lambda i: len(i['members']))

    # Dump warnings by severity. If severity.SKIP warnings are not dumpped,

    # the project anchors should be dumped through dumpskippedanchors.

  warn_patterns.sort(reverse=True, key=lambda i: len(i['members']))

  # Dump warnings by severity. If severity.SKIP warnings are not dumped,

  # the project anchors should be dumped through dump_skipped_anchors.

  for s in severity.kinds:

        dumpseverity(s)

    dumpfixed()

    dumphtmlepilogue()

    dump_severity(s)

  dump_fixed()

  dump_html_epilogue()





##### Functions to count warnings and dump csv file. #########################



def descriptionforcsv(cat):

    if cat['description'] == '':



def description_for_csv(cat):

  if not cat['description']:

    return '?'

  return cat['description']



def stringforcsv(s):



def string_for_csv(s):

  if ',' in s:

    return '"{}"'.format(s)

  return s



def countseverity(sev, kind):

  sum = 0

  for i in warnpatterns:

      if i['severity'] == sev and len(i['members']) > 0:



def count_severity(sev, kind):

  """Count warnings of given severity."""

  total = 0

  for i in warn_patterns:

    if i['severity'] == sev and i['members']:

      n = len(i['members'])

          sum += n

          warning = stringforcsv(kind + ': ' + descriptionforcsv(i))

      total += n

      warning = string_for_csv(kind + ': ' + description_for_csv(i))

      print '{},,{}'.format(n, warning)

      # print number of warnings for each project, ordered by project name.

      projects = i['projects'].keys()

      projects.sort()

      for p in projects:

        print '{},{},{}'.format(i['projects'][p], p, warning)

  print '{},,{}'.format(sum, kind + ' warnings')

  return sum

  print '{},,{}'.format(total, kind + ' warnings')

  return total





# dump number of warnings in csv format to stdout

def dumpcsv():

    sortwarnings()

def dump_csv():

  """Dump number of warnings in csv format to stdout."""

  sort_warnings()

  total = 0

  for s in severity.kinds:

        total += countseverity(s, severity.columnheader[s])

    total += count_severity(s, severity.column_headers[s])

  print '{},,{}'.format(total, 'All warnings')





parseinputfile()

parse_input_file()

if args.gencsv:

    dumpcsv()

  dump_csv()

else:

    dumphtml()
 
  dump_html()