| #!/usr/bin/python3 |
| # |
| # Copyright (C) 2016 The Android Open Source Project |
| # |
| # Licensed under the Apache License, Version 2.0 (the "License"); |
| # you may not use this file except in compliance with the License. |
| # You may obtain a copy of the License at |
| # |
| # http://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, software |
| # distributed under the License is distributed on an "AS IS" BASIS, |
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| # See the License for the specific language governing permissions and |
| # limitations under the License. |
| import argparse, collections, os, re, sys |
| |
| dir_of_this_script = os.path.dirname(os.path.realpath(__file__)) |
| |
| parser = argparse.ArgumentParser( |
| description="""USAGE: |
| Simplifies a build.log from hundreds of megabytes to <100 lines. Prints output to terminal. |
| Pass this script a filepath to parse. You should be able to type "python3 build_log_simplifier.py" |
| And then drag-and-drop a log file onto the terminal window to get its path. |
| |
| Sample usage: python3 development/build_log_simplifier.py Users/owengray/Desktop/build.log |
| """) |
| parser.add_argument("--validate", action="store_true", help="Validate that no unrecognized messages exist in the given log") |
| parser.add_argument("--update", action="store_true", help="Update our list of recognized messages to include all messages from the given log") |
| parser.add_argument("--gc", action="store_true", help="When generating a new exemptions file, exclude any exemptions that were not found in the given log. Only relevant with --update or --validate") |
| parser.add_argument("log_path", help="Filepath of log(s) to process", nargs="+") |
| |
| # a regexes_matcher can quickly identify which of a set of regexes matches a given text |
| class regexes_matcher(object): |
| def __init__(self, regexes): |
| self.regex_texts = regexes |
| self.children = None |
| self.matcher = None |
| |
| # returns a list of regexes that match the given text |
| def get_matching_regexes(self, text, expect_match=True): |
| if expect_match and len(self.regex_texts) > 1: |
| # If we already expect our matcher to match, we can directly jump to asking our children |
| return self.query_children_for_matching_regexes(text) |
| # It takes more time to match lots of regexes than to match one composite regex |
| # So, we try to match one composite regex first |
| if self.matches(text): |
| if len(self.regex_texts) > 1: |
| # At least one child regex matches, so we have to determine which ones |
| return self.query_children_for_matching_regexes(text) |
| else: |
| return self.regex_texts |
| # Our composite regex yielded no matches |
| return [] |
| |
| # queries our children for regexes that match <text> |
| def query_children_for_matching_regexes(self, text): |
| # Create children if they don't yet exist |
| self.ensure_split() |
| # query children and join their results |
| results = [] |
| for child in self.children: |
| results += child.get_matching_regexes(text, False) |
| return results |
| |
| # Returns the index of the first regex matching this string, or None of not found |
| def index_first_matching_regex(self, text): |
| if len(self.regex_texts) <= 1: |
| if len(self.regex_texts) == 0: |
| return None |
| if self.matches(text): |
| return 0 |
| return None |
| if not self.matches(text): |
| return None |
| self.ensure_split() |
| count = 0 |
| for child in self.children: |
| child_index = child.index_first_matching_regex(text) |
| if child_index is not None: |
| return count + child_index |
| count += len(child.regex_texts) |
| return None |
| |
| # Create children if they don't yet exist |
| def ensure_split(self): |
| if self.children is None: |
| # It takes more time to compile a longer regex, but it also takes more time to |
| # test lots of small regexes. |
| # In practice, this number of children seems to result in fast execution |
| num_children = min(len(self.regex_texts), 32) |
| child_start = 0 |
| self.children = [] |
| for i in range(num_children): |
| child_end = int(len(self.regex_texts) * (i + 1) / num_children) |
| self.children.append(regexes_matcher(self.regex_texts[child_start:child_end])) |
| child_start = child_end |
| |
| |
| def matches(self, text): |
| if self.matcher is None: |
| full_regex_text = "(?:" + ")|(?:".join(self.regex_texts) + ")" |
| self.matcher = re.compile(full_regex_text) |
| return self.matcher.fullmatch(text) |
| |
| |
| def print_failing_task_names(lines): |
| tasks_of_interest = [] |
| # first, find tasks of interest |
| for line in lines: |
| if line.startswith("Execution failed for task"): |
| tasks_of_interest.append(line.split("task '")[1][:-3]) |
| |
| print("Detected these failing tasks: " + str(tasks_of_interest)) |
| |
| def shorten_uninteresting_stack_frames(lines): |
| result = [] |
| prev_line_is_boring = False |
| for line in lines: |
| if line.startswith("\tat ") and not line.startswith("\tat androidx"): |
| # non-androidx stack frame |
| if not prev_line_is_boring: |
| result.append(line.replace("\n", "...\n")) |
| prev_line_is_boring = True |
| else: |
| result.append(line) |
| prev_line_is_boring = False |
| return result |
| |
| # Returns the path of the config file holding exemptions for deterministic/consistent output. |
| # These exemptions can be garbage collected via the `--gc` argument |
| def get_deterministic_exemptions_path(): |
| return os.path.join(dir_of_this_script, "messages.ignore") |
| |
| # Returns the path of the config file holding exemptions for nondetermistic/flaky output. |
| # These exemptions will not be garbage collected via the `--gc` argument |
| def get_flake_exemptions_path(): |
| return os.path.join(dir_of_this_script, "message-flakes.ignore") |
| |
| # Returns a regexes_matcher that matches what is described by our config file |
| # Ignores comments and ordering in our config file |
| def build_exemptions_matcher(config_lines): |
| config_lines = [line.replace("\n", "") for line in config_lines] |
| regexes = [] |
| for line in config_lines: |
| line = line.strip() |
| if line.startswith("#") or line == "": |
| # skip comments |
| continue |
| regexes.append(line) |
| if remove_control_characters(line) != line: |
| raise Exception("Unexpected control characters found in configuration line:\n\n " + |
| "'" + line + "'\n\n. This line is unexpected to match anything. Is this a copying mistake?") |
| |
| return regexes_matcher(sorted(regexes)) |
| |
| # Returns a regexes_matcher that matches the content of our config file |
| # Can match comments |
| # Respects ordering in the config |
| # This is used for editing the config file itself |
| def build_exemptions_code_matcher(config_lines): |
| config_lines = [line.strip() for line in config_lines] |
| regexes = [] |
| for line in config_lines: |
| line = line.strip() |
| if line == "": |
| continue |
| regexes.append(line) |
| return regexes_matcher(regexes) |
| |
| def remove_by_regexes(lines, config_lines, validate_no_duplicates): |
| fast_matcher = build_exemptions_matcher(config_lines) |
| result = [] |
| for line in lines: |
| stripped = line.strip() |
| matching_exemptions = fast_matcher.get_matching_regexes(stripped, expect_match=True) |
| if validate_no_duplicates and len(matching_exemptions) > 1: |
| print("") |
| print("build_log_simplifier.py: Invalid configuration: multiple message exemptions match the same message. Are some exemptions too broad?") |
| print("") |
| print("Line: '" + stripped + "'") |
| print("") |
| print(str(len(matching_exemptions)) + " Matching exemptions:") |
| for exemption_text in matching_exemptions: |
| print("'" + exemption_text + "'") |
| exit(1) |
| if len(matching_exemptions) < 1: |
| result.append(line) |
| return result |
| |
| def collapse_consecutive_blank_lines(lines): |
| result = [] |
| prev_blank = True |
| for line in lines: |
| if line.strip() == "": |
| if not prev_blank: |
| result.append(line) |
| prev_blank = True |
| else: |
| result.append(line) |
| prev_blank = False |
| return result |
| |
| def remove_trailing_blank_lines(lines): |
| while len(lines) > 0 and lines[-1].strip() == "": |
| del lines[-1] |
| return lines |
| |
| def extract_task_name(line): |
| prefix = "> Task " |
| if line.startswith(prefix): |
| return line[len(prefix):].strip() |
| return None |
| |
| def is_task_line(line): |
| return extract_task_name(line) is not None |
| |
| def extract_task_names(lines): |
| names = [] |
| for line in lines: |
| name = extract_task_name(line) |
| if name is not None and name not in names: |
| names.append(name) |
| return names |
| |
| # If a task has no output (or only blank output), this function removes the task (and its output) |
| # For example, turns this: |
| # > Task :a |
| # > Task :b |
| # some message |
| # |
| # into this: |
| # |
| # > Task :b |
| # some message |
| def collapse_tasks_having_no_output(lines): |
| result = [] |
| # When we see a task name, we might not emit it if it doesn't have any output |
| # This variable is that pending task name, or none if we have no pending task |
| pending_task = None |
| pending_blanks = [] |
| for line in lines: |
| is_section = is_task_line(line) or line.startswith("> Configure project ") or line.startswith("FAILURE: Build failed with an exception.") |
| if is_section: |
| pending_task = line |
| pending_blanks = [] |
| elif line.strip() == "": |
| # If we have a pending task and we found a blank line, then hold the blank line, |
| # and only output it if we later find some nonempty output |
| if pending_task is not None: |
| pending_blanks.append(line) |
| else: |
| result.append(line) |
| else: |
| # We found some nonempty output, now we emit any pending task names |
| if pending_task is not None: |
| result.append(pending_task) |
| result += pending_blanks |
| pending_task = None |
| pending_blanks = [] |
| result.append(line) |
| return result |
| |
| # Removes color characters and other ANSI control characters from this input |
| control_character_regex = re.compile(r""" |
| \x1B # Escape |
| (?: # 7-bit C1 Fe (except CSI) |
| [@-Z\\-_] |
| | # or [ for CSI, followed by a control sequence |
| \[ |
| [0-?]* # Parameters |
| [ -/]* # Intermediate bytes |
| [@-~] # End |
| ) |
| """, re.VERBOSE) |
| |
| def remove_control_characters(line): |
| return control_character_regex.sub("", line) |
| |
| # Removes strings from the input wherever they are found |
| # This list is less convenient than the .ignore files: |
| # This list doesn't get autosuggested additions |
| # This list isn't automatically garbage collected |
| # Users interested in seeing the exemption history probably won't think to look here |
| # This list does allow removing part of the text from a line and still validating the remainder of the line |
| # If this list eventually gets long we might want to make it easier to update |
| inline_ignores_regex = re.compile( |
| # b/300072778 |
| "Sharing is only supported for boot loader classes because bootstrap classpath has been appended" |
| ) |
| |
| def remove_inline_ignores(line): |
| return re.sub(inline_ignores_regex, "", line) |
| |
| # Normalizes some filepaths to more easily simplify/skip some messages |
| def normalize_paths(lines): |
| # get OUT_DIR, DIST_DIR, and the path of the root of the checkout |
| out_dir = None |
| dist_dir = None |
| checkout_dir = None |
| gradle_user_home = None |
| # we read checkout_root from the log file in case this build was run in a location, |
| # such as on a build server |
| out_marker = "OUT_DIR=" |
| dist_marker = "DIST_DIR=" |
| checkout_marker = "CHECKOUT=" |
| gradle_user_home_marker="GRADLE_USER_HOME=" |
| for line in lines: |
| if line.startswith(out_marker): |
| out_dir = line.split(out_marker)[1].strip() |
| continue |
| if line.startswith(dist_marker): |
| dist_dir = line.split(dist_marker)[1].strip() |
| continue |
| if line.startswith(checkout_marker): |
| checkout_dir = line.split(checkout_marker)[1].strip() |
| continue |
| if line.startswith(gradle_user_home_marker): |
| gradle_user_home = line.split(gradle_user_home_marker)[1].strip() |
| continue |
| if out_dir is not None and dist_dir is not None and checkout_dir is not None and gradle_user_home is not None: |
| break |
| |
| # Remove any mentions of these paths, and replace them with consistent values |
| # Make sure to put these paths in the correct order so that more-specific paths will |
| # be matched first |
| remove_paths = collections.OrderedDict() |
| if gradle_user_home is not None: |
| remove_paths[gradle_user_home] = "$GRADLE_USER_HOME" |
| if dist_dir is not None: |
| remove_paths[dist_dir] = "$DIST_DIR" |
| if out_dir is not None: |
| remove_paths[out_dir] = "$OUT_DIR" |
| if checkout_dir is not None: |
| remove_paths[checkout_dir + "/frameworks/support"] = "$SUPPORT" |
| remove_paths[checkout_dir] = "$CHECKOUT" |
| result = [] |
| for line in lines: |
| for path in remove_paths: |
| if path in line: |
| replacement = remove_paths[path] |
| line = line.replace(path + "/", replacement + "/") |
| line = line.replace(path, replacement) |
| result.append(line) |
| return result |
| |
| # Given a regex with hashes in it like ".gradle/caches/transforms-2/files-2.1/73f631f487bd87cfd8cb2aabafbac6a8", |
| # tries to return a more generalized regex like ".gradle/caches/transforms-2/files-2.1/[0-9a-f]{32}" |
| def generalize_hashes(message): |
| hash_matcher = "[0-9a-f]{32}" |
| return re.sub(hash_matcher, hash_matcher, message) |
| |
| # Given a regex with numbers in it like ".gradle/caches/transforms-2/files-2.1/73f631f487bd87cfd8cb2aabafbac6a8" |
| # tries to return a more generalized regex like ".gradle/caches/transforms-[0-9]*/files-[0-9]*.[0-9]*/73f631f487bd87cfd8cb2aabafbac6a8" |
| def generalize_numbers(message): |
| matcher = "[0-9]+" |
| generalized = re.sub(matcher, matcher, message) |
| # the above replacement corrupts strings of the form "[0-9a-f]{32}", so we fix them before returning |
| return generalized.replace("[[0-9]+-[0-9]+a-f]{[0-9]+}", "[0-9a-f]{32}") |
| |
| # Given a list of output messages and a list of existing exemption lines, |
| # generates a new list of exemption lines |
| def generate_suggested_exemptions(messages, config_lines, remove_unmatched_lines): |
| new_config = suggest_missing_exemptions(messages, config_lines) |
| if remove_unmatched_lines: |
| new_config = remove_unmatched_exemptions(messages, new_config) |
| return new_config |
| |
| # Given a list of output messages and a list of existing exemption lines, |
| # generates an augmented list of exemptions containing any necessary new exemptions |
| def suggest_missing_exemptions(messages, config_lines): |
| # given a message, finds the index of the existing exemption for that message, if any |
| existing_matcher = build_exemptions_code_matcher(config_lines) |
| # the index of the previously matched exemption |
| previous_found_index = -1 |
| # map from line index to list of lines to insert there |
| insertions_by_position = collections.defaultdict(lambda: []) |
| insertions_by_task_name = collections.OrderedDict() |
| # current task generating any subsequent output |
| pending_task_line = None |
| # new, suggested exemptions |
| new_suggestions = set() |
| # generate new suggestions |
| for line in messages: |
| line = line.strip() |
| if line == "": |
| continue |
| # save task name |
| is_section = False |
| if is_task_line(line) or line.startswith("> Configure project "): |
| # If a task creates output, we record its name |
| line = "# " + line |
| pending_task_line = line |
| is_section = True |
| # determine where to put task name |
| current_found_index = existing_matcher.index_first_matching_regex(line) |
| if current_found_index is not None: |
| # We already have a mention of this line |
| # We don't need to exempt it again, but this informs where to insert our next exemption |
| previous_found_index = current_found_index |
| pending_task_line = None |
| continue |
| # skip outputting task names for tasks that don't output anything |
| if is_section: |
| continue |
| |
| # escape message |
| escaped = re.escape(line) |
| escaped = escaped.replace("\ ", " ") # spaces don't need to be escaped |
| escaped = generalize_hashes(escaped) |
| escaped = generalize_numbers(escaped) |
| # confirm that we haven't already inserted this message |
| if escaped in new_suggestions: |
| continue |
| # insert this regex into an appropriate position |
| if pending_task_line is not None: |
| # We know which task this line came from, and it's a task that didn't previously make output |
| if pending_task_line not in insertions_by_task_name: |
| insertions_by_task_name[pending_task_line] = [] |
| insertions_by_task_name[pending_task_line].append(escaped) |
| else: |
| # This line of output didn't come from a new task |
| # So we append it after the previous line that we found |
| insertions_by_position[previous_found_index].append(escaped) |
| new_suggestions.add(escaped) |
| |
| # for each regex for which we chose a position in the file, insert it there |
| exemption_lines = [] |
| for i in range(len(existing_matcher.regex_texts)): |
| exemption_lines.append(existing_matcher.regex_texts[i]) |
| if i in insertions_by_position: |
| exemption_lines += insertions_by_position[i] |
| # for regexes that could not be assigned to a task, insert them next |
| if -1 in insertions_by_position: |
| exemption_lines += insertions_by_position[-1] |
| # for regexes that were simply assigned to certain task names, insert the there, grouped by task |
| for task_name in insertions_by_task_name: |
| exemption_lines.append(task_name) |
| exemption_lines += insertions_by_task_name[task_name] |
| return exemption_lines |
| |
| # Searches for config lines in <config_lines> that match no line in <messages> |
| # Create and returns a new list of config lines, which excludes unmatched lines and |
| # any corresponding comments |
| def remove_unmatched_exemptions(messages, config_lines): |
| existing_matcher = build_exemptions_matcher(config_lines) |
| matched_config_lines = set() |
| # find all of the regexes that match at least one message |
| for line in messages: |
| line = line.strip() |
| if line.startswith("#"): |
| continue |
| for regex in existing_matcher.get_matching_regexes(line): |
| matched_config_lines.add(regex) |
| # generate a new list of config lines |
| # keep config lines that were matched in the list of messages |
| # keep comments where there remains a matched config line before the next comment |
| # skip comments that were previously followed by other config lines that were deleted |
| result = [] |
| pending_comments = [] # comments that we haven't yet decided to keep or not |
| found_unused_line_after_comment = False |
| for line in config_lines: |
| if line.startswith("#"): |
| # We found a comment |
| if found_unused_line_after_comment: |
| # We found an unused config line more recently than the previous comment, |
| # and now we've found a new comment. |
| if len(pending_comments) > 0: |
| # We also haven't found any used config lines more recently than the previous comment |
| # Presumably these pending comments were intended to describe the lines that we're removing |
| # So, we skip emitting these pending comments too |
| pending_comments = [] |
| pending_comments.append(line) |
| found_unused_line_after_comment = False |
| continue |
| matched = (line in matched_config_lines) |
| if matched: |
| # If this config line is being used, then we keep its comments too |
| result += pending_comments |
| pending_comments = [] |
| result.append(line) |
| else: |
| found_unused_line_after_comment = True |
| # If there are any comments at the bottom of the file, then keep them too |
| if not found_unused_line_after_comment: |
| result += pending_comments |
| return result |
| |
| # opens a file and reads the lines in it |
| def readlines(path): |
| infile = open(path) |
| lines = infile.readlines() |
| infile.close() |
| return lines |
| |
| def writelines(path, lines): |
| destfile = open(path, 'w') |
| destfile.write("\n".join(lines)) |
| destfile.close() |
| |
| def main(): |
| arguments = parser.parse_args() |
| |
| # read each file |
| log_paths = arguments.log_path |
| all_lines = [] |
| for log_path in log_paths: |
| lines = readlines(log_path) |
| lines = [remove_control_characters(line) for line in lines] |
| lines = [remove_inline_ignores(line) for line in lines] |
| lines = normalize_paths(lines) |
| all_lines += lines |
| # load configuration |
| flake_exemption_regexes = readlines(get_flake_exemptions_path()) |
| deterministic_exemption_regexes = readlines(get_deterministic_exemptions_path()) |
| exemption_regexes = flake_exemption_regexes + deterministic_exemption_regexes |
| # load configuration |
| # remove lines we're not interested in |
| update = arguments.update or arguments.gc |
| validate = update or arguments.validate |
| interesting_lines = all_lines |
| if not validate: |
| print_failing_task_names(interesting_lines) |
| interesting_lines = remove_by_regexes(interesting_lines, exemption_regexes, validate) |
| interesting_lines = collapse_tasks_having_no_output(interesting_lines) |
| interesting_lines = collapse_consecutive_blank_lines(interesting_lines) |
| interesting_lines = remove_trailing_blank_lines(interesting_lines) |
| |
| # process results |
| if update: |
| if arguments.gc or len(interesting_lines) != 0: |
| update_path = get_deterministic_exemptions_path() |
| # filter out any inconsistently observed messages so we don't try to exempt them twice |
| all_lines = remove_by_regexes(all_lines, flake_exemption_regexes, validate) |
| # update the deterministic exemptions file based on the result |
| suggested = generate_suggested_exemptions(all_lines, deterministic_exemption_regexes, arguments.gc) |
| writelines(update_path, suggested) |
| print("build_log_simplifier.py updated exemptions " + update_path) |
| elif validate: |
| if len(interesting_lines) != 0: |
| print("") |
| print("=" * 80) |
| print("build_log_simplifier.py: Error: Found " + str(len(interesting_lines)) + " new lines of warning output!") |
| print("") |
| print("The new output:") |
| print(" " + " ".join(interesting_lines)) |
| print("") |
| print("To reproduce this failure:") |
| print(" Try $ ./gradlew -Pandroidx.validateNoUnrecognizedMessages --rerun-tasks " + " ".join(extract_task_names(interesting_lines))) |
| print("") |
| print("Instructions:") |
| print(" If you can fix these messages, do so.") |
| print(" If you cannot fix these messages, you may suppress them.") |
| print(" To automatically suppress new output from build server builds, run development/build_log_simplifier/update.sh") |
| print(" See also https://android.googlesource.com/platform/frameworks/support/+/androidx-main/development/build_log_simplifier/VALIDATION_FAILURE.md") |
| print("") |
| new_exemptions_path = log_paths[0] + ".ignore" |
| # filter out any inconsistently observed messages so we don't try to exempt them twice |
| all_lines = remove_by_regexes(all_lines, flake_exemption_regexes, validate) |
| # update deterministic exemptions file based on the result |
| suggested = generate_suggested_exemptions(all_lines, deterministic_exemption_regexes, arguments.gc) |
| writelines(new_exemptions_path, suggested) |
| print("Files:") |
| print(" Full Log : " + ",".join(log_paths)) |
| print(" Baseline : " + get_deterministic_exemptions_path()) |
| print(" Autogenerated new baseline : " + new_exemptions_path) |
| exit(1) |
| else: |
| interesting_lines = shorten_uninteresting_stack_frames(interesting_lines) |
| print("".join(interesting_lines)) |
| |
| if __name__ == "__main__": |
| main() |
