| # python3 |
| # Copyright (C) 2019 The Android Open Source Project |
| # |
| # Licensed under the Apache License, Version 2.0 (the "License"); |
| # you may not use this file except in compliance with the License. |
| # You may obtain a copy of the License at |
| # |
| # http://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, software |
| # distributed under the License is distributed on an "AS IS" BASIS, |
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| # See the License for the specific language governing permissions and |
| # limitations under the License. |
| |
| """Grep warnings messages and output HTML tables or warning counts in CSV. |
| |
| Default is to output warnings in HTML tables grouped by warning severity. |
| Use option --byproject to output tables grouped by source file projects. |
| Use option --gencsv to output warning counts in CSV format. |
| """ |
| |
| # List of important data structures and functions in this script. |
| # |
| # To parse and keep warning message in the input file: |
| # severity: classification of message severity |
| # warn_patterns: |
| # warn_patterns[w]['category'] tool that issued the warning, not used now |
| # warn_patterns[w]['description'] table heading |
| # warn_patterns[w]['members'] matched warnings from input |
| # warn_patterns[w]['patterns'] regular expressions to match warnings |
| # warn_patterns[w]['projects'][p] number of warnings of pattern w in p |
| # warn_patterns[w]['severity'] severity tuple |
| # project_list[p][0] project name |
| # project_list[p][1] regular expression to match a project path |
| # project_patterns[p] re.compile(project_list[p][1]) |
| # project_names[p] project_list[p][0] |
| # warning_messages array of each warning message, without source url |
| # warning_records array of [idx to warn_patterns, |
| # idx to project_names, |
| # idx to warning_messages] |
| # android_root |
| # platform_version |
| # target_product |
| # target_variant |
| # parse_input_file |
| # |
| # To emit html page of warning messages: |
| # flags: --byproject, --url, --separator |
| # Old stuff for static html components: |
| # html_script_style: static html scripts and styles |
| # htmlbig: |
| # dump_stats, dump_html_prologue, dump_html_epilogue: |
| # emit_buttons: |
| # dump_fixed |
| # sort_warnings: |
| # emit_stats_by_project: |
| # all_patterns, |
| # findproject, classify_warning |
| # dump_html |
| # |
| # New dynamic HTML page's static JavaScript data: |
| # Some data are copied from Python to JavaScript, to generate HTML elements. |
| # FlagURL args.url |
| # FlagSeparator args.separator |
| # SeverityColors: list of colors for all severity levels |
| # SeverityHeaders: list of headers for all severity levels |
| # SeverityColumnHeaders: list of column_headers for all severity levels |
| # ProjectNames: project_names, or project_list[*][0] |
| # WarnPatternsSeverity: warn_patterns[*]['severity'] |
| # WarnPatternsDescription: warn_patterns[*]['description'] |
| # WarningMessages: warning_messages |
| # Warnings: warning_records |
| # StatsHeader: warning count table header row |
| # StatsRows: array of warning count table rows |
| # |
| # New dynamic HTML page's dynamic JavaScript data: |
| # |
| # New dynamic HTML related function to emit data: |
| # escape_string, strip_escape_string, emit_warning_arrays |
| # emit_js_data(): |
| |
| from __future__ import print_function |
| import argparse |
| import cgi |
| import csv |
| import io |
| import multiprocessing |
| import os |
| import re |
| import signal |
| import sys |
| |
| # pylint:disable=relative-beyond-top-level |
| from . import cpp_warn_patterns |
| from . import java_warn_patterns |
| from . import make_warn_patterns |
| from . import other_warn_patterns |
| from . import tidy_warn_patterns |
| # pylint:disable=g-importing-member |
| from .android_project_list import project_list |
| from .severity import Severity |
| |
| parser = argparse.ArgumentParser(description='Convert a build log into HTML') |
| parser.add_argument('--csvpath', |
| help='Save CSV warning file to the passed absolute path', |
| default=None) |
| parser.add_argument('--gencsv', |
| help='Generate a CSV file with number of various warnings', |
| action='store_true', |
| default=False) |
| parser.add_argument('--byproject', |
| help='Separate warnings in HTML output by project names', |
| action='store_true', |
| default=False) |
| parser.add_argument('--url', |
| help='Root URL of an Android source code tree prefixed ' |
| 'before files in warnings') |
| parser.add_argument('--separator', |
| help='Separator between the end of a URL and the line ' |
| 'number argument. e.g. #') |
| parser.add_argument('--processes', |
| type=int, |
| default=multiprocessing.cpu_count(), |
| help='Number of parallel processes to process warnings') |
| parser.add_argument(dest='buildlog', metavar='build.log', |
| help='Path to build.log file') |
| args = parser.parse_args() |
| |
| warn_patterns = make_warn_patterns.warn_patterns |
| warn_patterns.extend(cpp_warn_patterns.warn_patterns) |
| warn_patterns.extend(java_warn_patterns.warn_patterns) |
| warn_patterns.extend(tidy_warn_patterns.warn_patterns) |
| warn_patterns.extend(other_warn_patterns.warn_patterns) |
| |
| project_patterns = [] |
| project_names = [] |
| warning_messages = [] |
| warning_records = [] |
| |
| |
| def initialize_arrays(): |
| """Complete global arrays before they are used.""" |
| global project_names, project_patterns |
| project_names = [p[0] for p in project_list] |
| project_patterns = [re.compile(p[1]) for p in project_list] |
| for w in warn_patterns: |
| w['members'] = [] |
| # Each warning pattern has a 'projects' dictionary, that |
| # maps a project name to number of warnings in that project. |
| w['projects'] = {} |
| |
| |
| initialize_arrays() |
| |
| |
| android_root = '' |
| platform_version = 'unknown' |
| target_product = 'unknown' |
| target_variant = 'unknown' |
| |
| |
| ##### Data and functions to dump html file. ################################## |
| |
| html_head_scripts = """\ |
| <script type="text/javascript"> |
| function expand(id) { |
| var e = document.getElementById(id); |
| var f = document.getElementById(id + "_mark"); |
| if (e.style.display == 'block') { |
| e.style.display = 'none'; |
| f.innerHTML = '⊕'; |
| } |
| else { |
| e.style.display = 'block'; |
| f.innerHTML = '⊖'; |
| } |
| }; |
| function expandCollapse(show) { |
| for (var id = 1; ; id++) { |
| var e = document.getElementById(id + ""); |
| var f = document.getElementById(id + "_mark"); |
| if (!e || !f) break; |
| e.style.display = (show ? 'block' : 'none'); |
| f.innerHTML = (show ? '⊖' : '⊕'); |
| } |
| }; |
| </script> |
| <style type="text/css"> |
| th,td{border-collapse:collapse; border:1px solid black;} |
| .button{color:blue;font-size:110%;font-weight:bolder;} |
| .bt{color:black;background-color:transparent;border:none;outline:none; |
| font-size:140%;font-weight:bolder;} |
| .c0{background-color:#e0e0e0;} |
| .c1{background-color:#d0d0d0;} |
| .t1{border-collapse:collapse; width:100%; border:1px solid black;} |
| </style> |
| <script src="https://www.gstatic.com/charts/loader.js"></script> |
| """ |
| |
| |
| def make_writer(output_stream): |
| |
| def writer(text): |
| return output_stream.write(text + '\n') |
| |
| return writer |
| |
| |
| def html_big(param): |
| return '<font size="+2">' + param + '</font>' |
| |
| |
| def dump_html_prologue(title, writer): |
| writer('<html>\n<head>') |
| writer('<title>' + title + '</title>') |
| writer(html_head_scripts) |
| emit_stats_by_project(writer) |
| writer('</head>\n<body>') |
| writer(html_big(title)) |
| writer('<p>') |
| |
| |
| def dump_html_epilogue(writer): |
| writer('</body>\n</head>\n</html>') |
| |
| |
| def sort_warnings(): |
| for i in warn_patterns: |
| i['members'] = sorted(set(i['members'])) |
| |
| |
| def emit_stats_by_project(writer): |
| """Dump a google chart table of warnings per project and severity.""" |
| # warnings[p][s] is number of warnings in project p of severity s. |
| # pylint:disable=g-complex-comprehension |
| warnings = {p: {s.value: 0 for s in Severity.levels} for p in project_names} |
| for i in warn_patterns: |
| # pytype: disable=attribute-error |
| s = i['severity'].value |
| # pytype: enable=attribute-error |
| for p in i['projects']: |
| warnings[p][s] += i['projects'][p] |
| |
| # total_by_project[p] is number of warnings in project p. |
| total_by_project = { |
| p: sum(warnings[p][s.value] for s in Severity.levels) |
| for p in project_names |
| } |
| |
| # total_by_severity[s] is number of warnings of severity s. |
| total_by_severity = { |
| s.value: sum(warnings[p][s.value] for p in project_names) |
| for s in Severity.levels |
| } |
| |
| # emit table header |
| stats_header = ['Project'] |
| for s in Severity.levels: |
| if total_by_severity[s.value]: |
| stats_header.append( |
| '<span style=\'background-color:{}\'>{}</span>'.format( |
| s.color, s.column_header)) |
| stats_header.append('TOTAL') |
| |
| # emit a row of warning counts per project, skip no-warning projects |
| total_all_projects = 0 |
| stats_rows = [] |
| for p in project_names: |
| if total_by_project[p]: |
| one_row = [p] |
| for s in Severity.levels: |
| if total_by_severity[s.value]: |
| one_row.append(warnings[p][s.value]) |
| one_row.append(total_by_project[p]) |
| stats_rows.append(one_row) |
| total_all_projects += total_by_project[p] |
| |
| # emit a row of warning counts per severity |
| total_all_severities = 0 |
| one_row = ['<b>TOTAL</b>'] |
| for s in Severity.levels: |
| if total_by_severity[s.value]: |
| one_row.append(total_by_severity[s.value]) |
| total_all_severities += total_by_severity[s.value] |
| one_row.append(total_all_projects) |
| stats_rows.append(one_row) |
| writer('<script>') |
| emit_const_string_array('StatsHeader', stats_header, writer) |
| emit_const_object_array('StatsRows', stats_rows, writer) |
| writer(draw_table_javascript) |
| writer('</script>') |
| |
| |
| def dump_stats(writer): |
| """Dump some stats about total number of warnings and such.""" |
| known = 0 |
| skipped = 0 |
| unknown = 0 |
| sort_warnings() |
| for i in warn_patterns: |
| if i['severity'] == Severity.UNMATCHED: |
| unknown += len(i['members']) |
| elif i['severity'] == Severity.SKIP: |
| skipped += len(i['members']) |
| else: |
| known += len(i['members']) |
| writer('Number of classified warnings: <b>' + str(known) + '</b><br>') |
| writer('Number of skipped warnings: <b>' + str(skipped) + '</b><br>') |
| writer('Number of unclassified warnings: <b>' + str(unknown) + '</b><br>') |
| total = unknown + known + skipped |
| extra_msg = '' |
| if total < 1000: |
| extra_msg = ' (low count may indicate incremental build)' |
| writer('Total number of warnings: <b>' + str(total) + '</b>' + extra_msg) |
| |
| |
| # New base table of warnings, [severity, warn_id, project, warning_message] |
| # Need buttons to show warnings in different grouping options. |
| # (1) Current, group by severity, id for each warning pattern |
| # sort by severity, warn_id, warning_message |
| # (2) Current --byproject, group by severity, |
| # id for each warning pattern + project name |
| # sort by severity, warn_id, project, warning_message |
| # (3) New, group by project + severity, |
| # id for each warning pattern |
| # sort by project, severity, warn_id, warning_message |
| def emit_buttons(writer): |
| writer('<button class="button" onclick="expandCollapse(1);">' |
| 'Expand all warnings</button>\n' |
| '<button class="button" onclick="expandCollapse(0);">' |
| 'Collapse all warnings</button>\n' |
| '<button class="button" onclick="groupBySeverity();">' |
| 'Group warnings by severity</button>\n' |
| '<button class="button" onclick="groupByProject();">' |
| 'Group warnings by project</button><br>') |
| |
| |
| def all_patterns(category): |
| patterns = '' |
| for i in category['patterns']: |
| patterns += i |
| patterns += ' / ' |
| return patterns |
| |
| |
| def dump_fixed(writer): |
| """Show which warnings no longer occur.""" |
| anchor = 'fixed_warnings' |
| mark = anchor + '_mark' |
| writer('\n<br><p style="background-color:lightblue"><b>' |
| '<button id="' + mark + '" ' |
| 'class="bt" onclick="expand(\'' + anchor + '\');">' |
| '⊕</button> Fixed warnings. ' |
| 'No more occurrences. Please consider turning these into ' |
| 'errors if possible, before they are reintroduced in to the build' |
| ':</b></p>') |
| writer('<blockquote>') |
| fixed_patterns = [] |
| for i in warn_patterns: |
| if not i['members']: |
| fixed_patterns.append(i['description'] + ' (' + all_patterns(i) + ')') |
| fixed_patterns = sorted(fixed_patterns) |
| writer('<div id="' + anchor + '" style="display:none;"><table>') |
| cur_row_class = 0 |
| for text in fixed_patterns: |
| cur_row_class = 1 - cur_row_class |
| # remove last '\n' |
| t = text[:-1] if text[-1] == '\n' else text |
| writer('<tr><td class="c' + str(cur_row_class) + '">' + t + '</td></tr>') |
| writer('</table></div>') |
| writer('</blockquote>') |
| |
| |
| def find_project_index(line): |
| for p in range(len(project_patterns)): |
| if project_patterns[p].match(line): |
| return p |
| return -1 |
| |
| |
| def classify_one_warning(line, results): |
| """Classify one warning line.""" |
| for i in range(len(warn_patterns)): |
| w = warn_patterns[i] |
| for cpat in w['compiled_patterns']: |
| # pytype: disable=attribute-error |
| if cpat.match(line): |
| p = find_project_index(line) |
| results.append([line, i, p]) |
| return |
| else: |
| # If we end up here, there was a problem parsing the log |
| # probably caused by 'make -j' mixing the output from |
| # 2 or more concurrent compiles |
| pass |
| # pytype: enable=attribute-error |
| |
| |
| def classify_warnings(lines): |
| results = [] |
| for line in lines: |
| classify_one_warning(line, results) |
| # After the main work, ignore all other signals to a child process, |
| # to avoid bad warning/error messages from the exit clean-up process. |
| if args.processes > 1: |
| signal.signal(signal.SIGTERM, lambda *args: sys.exit(-signal.SIGTERM)) |
| return results |
| |
| |
| def parallel_classify_warnings(warning_lines, parallel_process): |
| """Classify all warning lines with num_cpu parallel processes.""" |
| num_cpu = args.processes |
| if num_cpu > 1: |
| groups = [[] for x in range(num_cpu)] |
| i = 0 |
| for x in warning_lines: |
| groups[i].append(x) |
| i = (i + 1) % num_cpu |
| group_results = parallel_process(num_cpu, classify_warnings, groups) |
| else: |
| group_results = [classify_warnings(warning_lines)] |
| |
| for result in group_results: |
| for line, pattern_idx, project_idx in result: |
| pattern = warn_patterns[pattern_idx] |
| pattern['members'].append(line) |
| message_idx = len(warning_messages) |
| warning_messages.append(line) |
| warning_records.append([pattern_idx, project_idx, message_idx]) |
| pname = '???' if project_idx < 0 else project_names[project_idx] |
| # Count warnings by project. |
| if pname in pattern['projects']: |
| pattern['projects'][pname] += 1 |
| else: |
| pattern['projects'][pname] = 1 |
| |
| |
| def find_warn_py_and_android_root(path): |
| """Set and return android_root path if it is found.""" |
| global android_root |
| parts = path.split('/') |
| for idx in reversed(range(2, len(parts))): |
| root_path = '/'.join(parts[:idx]) |
| # Android root directory should contain this script. |
| if os.path.exists(root_path + '/build/make/tools/warn.py'): |
| android_root = root_path |
| return True |
| return False |
| |
| |
| def find_android_root(): |
| """Guess android_root from common prefix of file paths.""" |
| # Use the longest common prefix of the absolute file paths |
| # of the first 10000 warning messages as the android_root. |
| global android_root |
| warning_lines = set() |
| warning_pattern = re.compile('^/[^ ]*/[^ ]*: warning: .*') |
| count = 0 |
| infile = io.open(args.buildlog, mode='r', encoding='utf-8') |
| for line in infile: |
| if warning_pattern.match(line): |
| warning_lines.add(line) |
| count += 1 |
| if count > 9999: |
| break |
| # Try to find warn.py and use its location to find |
| # the source tree root. |
| if count < 100: |
| path = os.path.normpath(re.sub(':.*$', '', line)) |
| if find_warn_py_and_android_root(path): |
| return |
| # Do not use common prefix of a small number of paths. |
| if count > 10: |
| # pytype: disable=wrong-arg-types |
| root_path = os.path.commonprefix(warning_lines) |
| # pytype: enable=wrong-arg-types |
| if len(root_path) > 2 and root_path[len(root_path) - 1] == '/': |
| android_root = root_path[:-1] |
| |
| |
| def remove_android_root_prefix(path): |
| """Remove android_root prefix from path if it is found.""" |
| if path.startswith(android_root): |
| return path[1 + len(android_root):] |
| else: |
| return path |
| |
| |
| def normalize_path(path): |
| """Normalize file path relative to android_root.""" |
| # If path is not an absolute path, just normalize it. |
| path = os.path.normpath(path) |
| # Remove known prefix of root path and normalize the suffix. |
| if path[0] == '/' and android_root: |
| return remove_android_root_prefix(path) |
| return path |
| |
| |
| def normalize_warning_line(line): |
| """Normalize file path relative to android_root in a warning line.""" |
| # replace fancy quotes with plain ol' quotes |
| line = re.sub(u'[\u2018\u2019]', '\'', line) |
| # replace non-ASCII chars to spaces |
| line = re.sub(u'[^\x00-\x7f]', ' ', line) |
| line = line.strip() |
| first_column = line.find(':') |
| if first_column > 0: |
| return normalize_path(line[:first_column]) + line[first_column:] |
| else: |
| return line |
| |
| |
| def parse_input_file(infile): |
| """Parse input file, collect parameters and warning lines.""" |
| global android_root |
| global platform_version |
| global target_product |
| global target_variant |
| line_counter = 0 |
| |
| # rustc warning messages have two lines that should be combined: |
| # warning: description |
| # --> file_path:line_number:column_number |
| # Some warning messages have no file name: |
| # warning: macro replacement list ... [bugprone-macro-parentheses] |
| # Some makefile warning messages have no line number: |
| # some/path/file.mk: warning: description |
| # C/C++ compiler warning messages have line and column numbers: |
| # some/path/file.c:line_number:column_number: warning: description |
| warning_pattern = re.compile('(^[^ ]*/[^ ]*: warning: .*)|(^warning: .*)') |
| warning_without_file = re.compile('^warning: .*') |
| rustc_file_position = re.compile('^[ ]+--> [^ ]*/[^ ]*:[0-9]+:[0-9]+') |
| |
| # Collect all warnings into the warning_lines set. |
| warning_lines = set() |
| prev_warning = '' |
| for line in infile: |
| if prev_warning: |
| if rustc_file_position.match(line): |
| # must be a rustc warning, combine 2 lines into one warning |
| line = line.strip().replace('--> ', '') + ': ' + prev_warning |
| warning_lines.add(normalize_warning_line(line)) |
| prev_warning = '' |
| continue |
| # add prev_warning, and then process the current line |
| prev_warning = 'unknown_source_file: ' + prev_warning |
| warning_lines.add(normalize_warning_line(prev_warning)) |
| prev_warning = '' |
| |
| if warning_pattern.match(line): |
| if warning_without_file.match(line): |
| # save this line and combine it with the next line |
| prev_warning = line |
| else: |
| warning_lines.add(normalize_warning_line(line)) |
| continue |
| |
| if line_counter < 100: |
| # save a little bit of time by only doing this for the first few lines |
| line_counter += 1 |
| m = re.search('(?<=^PLATFORM_VERSION=).*', line) |
| if m is not None: |
| platform_version = m.group(0) |
| m = re.search('(?<=^TARGET_PRODUCT=).*', line) |
| if m is not None: |
| target_product = m.group(0) |
| m = re.search('(?<=^TARGET_BUILD_VARIANT=).*', line) |
| if m is not None: |
| target_variant = m.group(0) |
| m = re.search('.* TOP=([^ ]*) .*', line) |
| if m is not None: |
| android_root = m.group(1) |
| return warning_lines |
| |
| |
| # Return s with escaped backslash and quotation characters. |
| def escape_string(s): |
| # pytype: disable=attribute-error |
| return s.replace('\\', '\\\\').replace('"', '\\"') |
| # pytype: enable=attribute-error |
| |
| |
| # Return s without trailing '\n' and escape the quotation characters. |
| def strip_escape_string(s): |
| if not s: |
| return s |
| s = s[:-1] if s[-1] == '\n' else s |
| return escape_string(s) |
| |
| |
| def emit_warning_array(name, writer): |
| writer('var warning_{} = ['.format(name)) |
| for i in range(len(warn_patterns)): |
| writer('{},'.format(warn_patterns[i][name])) |
| writer('];') |
| |
| |
| def emit_warning_arrays(writer): |
| emit_warning_array('severity', writer) |
| writer('var warning_description = [') |
| for i in range(len(warn_patterns)): |
| if warn_patterns[i]['members']: |
| writer('"{}",'.format(escape_string(warn_patterns[i]['description']))) |
| else: |
| writer('"",') # no such warning |
| writer('];') |
| |
| |
| scripts_for_warning_groups = """ |
| function compareMessages(x1, x2) { // of the same warning type |
| return (WarningMessages[x1[2]] <= WarningMessages[x2[2]]) ? -1 : 1; |
| } |
| function byMessageCount(x1, x2) { |
| return x2[2] - x1[2]; // reversed order |
| } |
| function bySeverityMessageCount(x1, x2) { |
| // orer by severity first |
| if (x1[1] != x2[1]) |
| return x1[1] - x2[1]; |
| return byMessageCount(x1, x2); |
| } |
| const ParseLinePattern = /^([^ :]+):(\\d+):(.+)/; |
| function addURL(line) { |
| if (FlagURL == "") return line; |
| if (FlagSeparator == "") { |
| return line.replace(ParseLinePattern, |
| "<a target='_blank' href='" + FlagURL + "/$1'>$1</a>:$2:$3"); |
| } |
| return line.replace(ParseLinePattern, |
| "<a target='_blank' href='" + FlagURL + "/$1" + FlagSeparator + |
| "$2'>$1:$2</a>:$3"); |
| } |
| function createArrayOfDictionaries(n) { |
| var result = []; |
| for (var i=0; i<n; i++) result.push({}); |
| return result; |
| } |
| function groupWarningsBySeverity() { |
| // groups is an array of dictionaries, |
| // each dictionary maps from warning type to array of warning messages. |
| var groups = createArrayOfDictionaries(SeverityColors.length); |
| for (var i=0; i<Warnings.length; i++) { |
| var w = Warnings[i][0]; |
| var s = WarnPatternsSeverity[w]; |
| var k = w.toString(); |
| if (!(k in groups[s])) |
| groups[s][k] = []; |
| groups[s][k].push(Warnings[i]); |
| } |
| return groups; |
| } |
| function groupWarningsByProject() { |
| var groups = createArrayOfDictionaries(ProjectNames.length); |
| for (var i=0; i<Warnings.length; i++) { |
| var w = Warnings[i][0]; |
| var p = Warnings[i][1]; |
| var k = w.toString(); |
| if (!(k in groups[p])) |
| groups[p][k] = []; |
| groups[p][k].push(Warnings[i]); |
| } |
| return groups; |
| } |
| var GlobalAnchor = 0; |
| function createWarningSection(header, color, group) { |
| var result = ""; |
| var groupKeys = []; |
| var totalMessages = 0; |
| for (var k in group) { |
| totalMessages += group[k].length; |
| groupKeys.push([k, WarnPatternsSeverity[parseInt(k)], group[k].length]); |
| } |
| groupKeys.sort(bySeverityMessageCount); |
| for (var idx=0; idx<groupKeys.length; idx++) { |
| var k = groupKeys[idx][0]; |
| var messages = group[k]; |
| var w = parseInt(k); |
| var wcolor = SeverityColors[WarnPatternsSeverity[w]]; |
| var description = WarnPatternsDescription[w]; |
| if (description.length == 0) |
| description = "???"; |
| GlobalAnchor += 1; |
| result += "<table class='t1'><tr bgcolor='" + wcolor + "'><td>" + |
| "<button class='bt' id='" + GlobalAnchor + "_mark" + |
| "' onclick='expand(\\"" + GlobalAnchor + "\\");'>" + |
| "⊕</button> " + |
| description + " (" + messages.length + ")</td></tr></table>"; |
| result += "<div id='" + GlobalAnchor + |
| "' style='display:none;'><table class='t1'>"; |
| var c = 0; |
| messages.sort(compareMessages); |
| for (var i=0; i<messages.length; i++) { |
| result += "<tr><td class='c" + c + "'>" + |
| addURL(WarningMessages[messages[i][2]]) + "</td></tr>"; |
| c = 1 - c; |
| } |
| result += "</table></div>"; |
| } |
| if (result.length > 0) { |
| return "<br><span style='background-color:" + color + "'><b>" + |
| header + ": " + totalMessages + |
| "</b></span><blockquote><table class='t1'>" + |
| result + "</table></blockquote>"; |
| |
| } |
| return ""; // empty section |
| } |
| function generateSectionsBySeverity() { |
| var result = ""; |
| var groups = groupWarningsBySeverity(); |
| for (s=0; s<SeverityColors.length; s++) { |
| result += createWarningSection(SeverityHeaders[s], SeverityColors[s], |
| groups[s]); |
| } |
| return result; |
| } |
| function generateSectionsByProject() { |
| var result = ""; |
| var groups = groupWarningsByProject(); |
| for (i=0; i<groups.length; i++) { |
| result += createWarningSection(ProjectNames[i], 'lightgrey', groups[i]); |
| } |
| return result; |
| } |
| function groupWarnings(generator) { |
| GlobalAnchor = 0; |
| var e = document.getElementById("warning_groups"); |
| e.innerHTML = generator(); |
| } |
| function groupBySeverity() { |
| groupWarnings(generateSectionsBySeverity); |
| } |
| function groupByProject() { |
| groupWarnings(generateSectionsByProject); |
| } |
| """ |
| |
| |
| # Emit a JavaScript const string |
| def emit_const_string(name, value, writer): |
| writer('const ' + name + ' = "' + escape_string(value) + '";') |
| |
| |
| # Emit a JavaScript const integer array. |
| def emit_const_int_array(name, array, writer): |
| writer('const ' + name + ' = [') |
| for n in array: |
| writer(str(n) + ',') |
| writer('];') |
| |
| |
| # Emit a JavaScript const string array. |
| def emit_const_string_array(name, array, writer): |
| writer('const ' + name + ' = [') |
| for s in array: |
| writer('"' + strip_escape_string(s) + '",') |
| writer('];') |
| |
| |
| # Emit a JavaScript const string array for HTML. |
| def emit_const_html_string_array(name, array, writer): |
| writer('const ' + name + ' = [') |
| for s in array: |
| # Not using html.escape yet, to work for both python 2 and 3, |
| # until all users switch to python 3. |
| # pylint:disable=deprecated-method |
| writer('"' + cgi.escape(strip_escape_string(s)) + '",') |
| writer('];') |
| |
| |
| # Emit a JavaScript const object array. |
| def emit_const_object_array(name, array, writer): |
| writer('const ' + name + ' = [') |
| for x in array: |
| writer(str(x) + ',') |
| writer('];') |
| |
| |
| def emit_js_data(writer): |
| """Dump dynamic HTML page's static JavaScript data.""" |
| emit_const_string('FlagURL', |
| args.url if args.url else '', writer) |
| emit_const_string('FlagSeparator', |
| args.separator if args.separator else '', writer) |
| emit_const_string_array('SeverityColors', |
| [s.color for s in Severity.levels], writer) |
| emit_const_string_array('SeverityHeaders', |
| [s.header for s in Severity.levels], writer) |
| emit_const_string_array('SeverityColumnHeaders', |
| [s.column_header for s in Severity.levels], writer) |
| emit_const_string_array('ProjectNames', project_names, writer) |
| # pytype: disable=attribute-error |
| emit_const_int_array('WarnPatternsSeverity', |
| [w['severity'].value for w in warn_patterns], writer) |
| # pytype: enable=attribute-error |
| emit_const_html_string_array('WarnPatternsDescription', |
| [w['description'] for w in warn_patterns], |
| writer) |
| emit_const_html_string_array('WarningMessages', warning_messages, writer) |
| emit_const_object_array('Warnings', warning_records, writer) |
| |
| draw_table_javascript = """ |
| google.charts.load('current', {'packages':['table']}); |
| google.charts.setOnLoadCallback(drawTable); |
| function drawTable() { |
| var data = new google.visualization.DataTable(); |
| data.addColumn('string', StatsHeader[0]); |
| for (var i=1; i<StatsHeader.length; i++) { |
| data.addColumn('number', StatsHeader[i]); |
| } |
| data.addRows(StatsRows); |
| for (var i=0; i<StatsRows.length; i++) { |
| for (var j=0; j<StatsHeader.length; j++) { |
| data.setProperty(i, j, 'style', 'border:1px solid black;'); |
| } |
| } |
| var table = new google.visualization.Table( |
| document.getElementById('stats_table')); |
| table.draw(data, {allowHtml: true, alternatingRowStyle: true}); |
| } |
| """ |
| |
| |
| def dump_html(output_stream): |
| """Dump the html output to output_stream.""" |
| writer = make_writer(output_stream) |
| dump_html_prologue('Warnings for ' + platform_version + ' - ' + |
| target_product + ' - ' + target_variant, writer) |
| dump_stats(writer) |
| writer('<br><div id="stats_table"></div><br>') |
| writer('\n<script>') |
| emit_js_data(writer) |
| writer(scripts_for_warning_groups) |
| writer('</script>') |
| emit_buttons(writer) |
| # Warning messages are grouped by severities or project names. |
| writer('<br><div id="warning_groups"></div>') |
| if args.byproject: |
| writer('<script>groupByProject();</script>') |
| else: |
| writer('<script>groupBySeverity();</script>') |
| dump_fixed(writer) |
| dump_html_epilogue(writer) |
| |
| |
| ##### Functions to count warnings and dump csv file. ######################### |
| |
| |
| def description_for_csv(category): |
| if not category['description']: |
| return '?' |
| return category['description'] |
| |
| |
| def count_severity(writer, sev, kind): |
| """Count warnings of given severity.""" |
| total = 0 |
| for i in warn_patterns: |
| if i['severity'] == sev and i['members']: |
| n = len(i['members']) |
| total += n |
| warning = kind + ': ' + description_for_csv(i) |
| writer.writerow([n, '', warning]) |
| # print number of warnings for each project, ordered by project name. |
| # pytype: disable=attribute-error |
| projects = sorted(i['projects'].keys()) |
| # pytype: enable=attribute-error |
| for p in projects: |
| writer.writerow([i['projects'][p], p, warning]) |
| writer.writerow([total, '', kind + ' warnings']) |
| |
| return total |
| |
| |
| # dump number of warnings in csv format to stdout |
| def dump_csv(writer): |
| """Dump number of warnings in csv format to stdout.""" |
| sort_warnings() |
| total = 0 |
| for s in Severity.levels: |
| if s != Severity.SEVERITY_UNKNOWN: |
| total += count_severity(writer, s, s.column_header) |
| writer.writerow([total, '', 'All warnings']) |
| |
| |
| def common_main(parallel_process): |
| """Real main function to classify warnings and generate .html file.""" |
| find_android_root() |
| # We must use 'utf-8' codec to parse some non-ASCII code in warnings. |
| warning_lines = parse_input_file( |
| io.open(args.buildlog, mode='r', encoding='utf-8')) |
| parallel_classify_warnings(warning_lines, parallel_process) |
| # If a user pases a csv path, save the fileoutput to the path |
| # If the user also passed gencsv write the output to stdout |
| # If the user did not pass gencsv flag dump the html report to stdout. |
| if args.csvpath: |
| with open(args.csvpath, 'w') as f: |
| dump_csv(csv.writer(f, lineterminator='\n')) |
| if args.gencsv: |
| dump_csv(csv.writer(sys.stdout, lineterminator='\n')) |
| else: |
| dump_html(sys.stdout) |
