blob: 07a51c39aaefc0d829406a5b8d5a43dc190177a4 [file] [log] [blame]
#!/usr/bin/env python3
#
# Copyright (C) 2021 The Android Open Source Project
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""A json-module-graph postprocessing script to generate a bp2build progress tracker.
Usage:
./bp2build-progress.py [report|graph] -m <module name>
Example:
To generate a report on the `adbd` module, run:
./bp2build-progress report -m adbd
To generate a graph on the `adbd` module, run:
./bp2build-progress graph -m adbd > graph.in && dot -Tpng -o graph.png
graph.in
"""
import argparse
import collections
import datetime
import dependency_analysis
import os.path
import queue
import subprocess
import sys
_ModuleInfo = collections.namedtuple("_ModuleInfo", [
"name",
"kind",
"dirname",
])
_ReportData = collections.namedtuple("_ReportData", [
"input_module",
"all_unconverted_modules",
"blocked_modules",
"dirs_with_unconverted_modules",
"kind_of_unconverted_modules",
"converted",
])
def combine_report_data(data):
ret = _ReportData(
input_module=set(),
all_unconverted_modules=collections.defaultdict(set),
blocked_modules=collections.defaultdict(set),
dirs_with_unconverted_modules=set(),
kind_of_unconverted_modules=set(),
converted=set(),
)
for item in data:
ret.input_module.add(item.input_module)
for key, value in item.all_unconverted_modules.items():
ret.all_unconverted_modules[key].update(value)
for key, value in item.blocked_modules.items():
ret.blocked_modules[key].update(value)
ret.dirs_with_unconverted_modules.update(item.dirs_with_unconverted_modules)
ret.kind_of_unconverted_modules.update(item.kind_of_unconverted_modules)
if len(ret.converted) == 0:
ret.converted.update(item.converted)
return ret
# Generate a dot file containing the transitive closure of the module.
def generate_dot_file(modules, converted, module):
DOT_TEMPLATE = """
digraph mygraph {{
node [shape=box];
%s
}}
"""
make_node = lambda module, color: \
('"{name}" [label="{name}\\n{kind}" color=black, style=filled, '
"fillcolor={color}]").format(name=module.name, kind=module.kind, color=color)
make_edge = lambda module, dep: \
'"%s" -> "%s"' % (module.name, dep)
# Check that all modules in the argument are in the list of converted modules
all_converted = lambda modules: all(map(lambda m: m in converted, modules))
dot_entries = []
for module, deps in modules.items():
if module.name in converted:
# Skip converted modules (nodes)
continue
elif module.name not in converted:
if all_converted(deps):
dot_entries.append(make_node(module, "yellow"))
else:
dot_entries.append(make_node(module, "tomato"))
# Print all edges for this module
for dep in list(deps):
# Skip converted deps (edges)
if dep not in converted:
dot_entries.append(make_edge(module, dep))
print(DOT_TEMPLATE % "\n ".join(dot_entries))
# Generate a report for each module in the transitive closure, and the blockers for each module
def generate_report_data(modules, converted, input_module):
# Map of [number of unconverted deps] to list of entries,
# with each entry being the string: "<module>: <comma separated list of unconverted modules>"
blocked_modules = collections.defaultdict(set)
# Map of unconverted modules to the modules they're blocking
# (i.e. reverse deps)
all_unconverted_modules = collections.defaultdict(set)
dirs_with_unconverted_modules = set()
kind_of_unconverted_modules = set()
for module, deps in sorted(modules.items()):
unconverted_deps = set(dep for dep in deps if dep not in converted)
for dep in unconverted_deps:
all_unconverted_modules[dep].add(module)
unconverted_count = len(unconverted_deps)
if module.name not in converted:
report_entry = "{name} [{kind}] [{dirname}]: {unconverted_deps}".format(
name=module.name,
kind=module.kind,
dirname=module.dirname,
unconverted_deps=", ".join(sorted(unconverted_deps)))
blocked_modules[unconverted_count].add(report_entry)
dirs_with_unconverted_modules.add(module.dirname)
kind_of_unconverted_modules.add(module.kind)
return _ReportData(
input_module=input_module,
all_unconverted_modules=all_unconverted_modules,
blocked_modules=blocked_modules,
dirs_with_unconverted_modules=dirs_with_unconverted_modules,
kind_of_unconverted_modules=kind_of_unconverted_modules,
converted=converted,
)
def generate_report(report_data):
report_lines = []
input_modules = sorted(report_data.input_module)
report_lines.append("# bp2build progress report for: %s\n" % input_modules)
report_lines.append("Ignored module types: %s\n" %
sorted(dependency_analysis.IGNORED_KINDS))
report_lines.append("# Transitive dependency closure:")
for count, modules in sorted(report_data.blocked_modules.items()):
report_lines.append("\n%d unconverted deps remaining:" % count)
for module_string in sorted(modules):
report_lines.append(" " + module_string)
report_lines.append("\n")
report_lines.append("# Unconverted deps of {}:\n".format(input_modules))
for count, dep in sorted(
((len(unconverted), dep)
for dep, unconverted in report_data.all_unconverted_modules.items()),
reverse=True):
report_lines.append("%s: blocking %d modules" % (dep, count))
report_lines.append("\n")
report_lines.append("# Dirs with unconverted modules:\n\n{}".format("\n".join(
sorted(report_data.dirs_with_unconverted_modules))))
report_lines.append("\n")
report_lines.append("# Kinds with unconverted modules:\n\n{}".format(
"\n".join(sorted(report_data.kind_of_unconverted_modules))))
report_lines.append("\n")
report_lines.append("# Converted modules:\n\n%s" %
"\n".join(sorted(report_data.converted)))
report_lines.append("\n")
report_lines.append(
"Generated by: https://cs.android.com/android/platform/superproject/+/master:build/bazel/scripts/bp2build-progress/bp2build-progress.py"
)
report_lines.append("Generated at: %s" %
datetime.datetime.now().strftime("%Y-%m-%dT%H:%M:%S %z"))
print("\n".join(report_lines))
def adjacency_list_from_json(module_graph, ignore_by_name, top_level_module):
def filter_by_name(json):
return json["Name"] == top_level_module
module_adjacency_list = collections.defaultdict(set)
def collect_dependencies(module, deps_names):
module_info = _ModuleInfo(
name=module["Name"],
kind=module["Type"],
dirname=os.path.dirname(module["Blueprint"]))
module_adjacency_list[module_info].update(deps_names)
dependency_analysis.module_graph_from_json(module_graph, ignore_by_name,
filter_by_name,
collect_dependencies)
return module_adjacency_list
def bazel_target_to_dir(full_target):
dirname, _ = full_target.split(":")
return dirname[2:]
def adjacency_list_from_queryview_xml(module_graph, ignore_by_name,
top_level_module):
# The set of ignored modules. These modules (and their dependencies) are
# not shown in the graph or report.
ignored = set()
# A map of module name to ModuleInfo
name_to_info = dict()
# queryview embeds variant in long name, keep a map of the name with vaiarnt
# to just name
name_with_variant_to_name = dict()
module_graph_map = dict()
q = queue.Queue()
for module in module_graph:
ignore = False
if module.tag != "rule":
continue
kind = module.attrib["class"]
name_with_variant = module.attrib["name"]
name = None
variant = ""
for attr in module:
attr_name = attr.attrib["name"]
if attr_name == "soong_module_name":
name = attr.attrib["value"]
elif attr_name == "soong_module_variant":
variant = attr.attrib["value"]
elif attr_name == "soong_module_type" and kind == "generic_soong_module":
kind = attr.attrib["value"]
# special handling for filegroup srcs, if a source has the same name as
# the module, we don't convert it
elif kind == "filegroup" and attr_name == "srcs":
for item in attr:
if item.attrib["value"] == name:
ignore = True
if name in ignore_by_name:
ignore = True
if dependency_analysis.ignore_kind(
kind, extra_kinds=dependency_analysis.QUERYVIEW_IGNORE_KINDS
) or variant.startswith("windows") or ignore:
ignored.add(name_with_variant)
else:
if name == top_level_module:
q.put(name_with_variant)
name_with_variant_to_name.setdefault(name_with_variant, name)
name_to_info.setdefault(
name,
_ModuleInfo(
name=name,
kind=kind,
dirname=bazel_target_to_dir(name_with_variant),
))
module_graph_map[name_with_variant] = module
# An adjacency list for all modules in the transitive closure, excluding ignored modules.
module_adjacency_list = {}
visited = set()
while not q.empty():
name_with_variant = q.get()
module = module_graph_map[name_with_variant]
if module.tag != "rule":
continue
visited.add(name_with_variant)
if name_with_variant in ignored:
continue
name = name_with_variant_to_name[name_with_variant]
module_info = name_to_info[name]
module_adjacency_list.setdefault(module_info, set())
for attr in module:
if attr.tag != "rule-input":
continue
dep_name_with_variant = attr.attrib["name"]
if dep_name_with_variant in ignored:
continue
dep_name = name_with_variant_to_name[dep_name_with_variant]
if dep_name_with_variant not in visited:
q.put(dep_name_with_variant)
if name != dep_name:
module_adjacency_list[module_info].add(dep_name)
return module_adjacency_list
def get_module_adjacency_list(top_level_module, use_queryview, ignore_by_name):
# The main module graph containing _all_ modules in the Soong build,
# and the list of converted modules.
try:
module_graph = dependency_analysis.get_queryview_module_info(
top_level_module
) if use_queryview else dependency_analysis.get_json_module_info(
top_level_module)
converted = dependency_analysis.get_bp2build_converted_modules()
except subprocess.CalledProcessError as err:
output = err.output.decode("utf-8") if err.output else ""
stderr = err.stderr.decode("utf-8") if err.stderr else ""
err_msg = """Error running: '{cmd}':"
Output:
{output}
Error:
{stderr}""".format(
cmd=" ".join(err.cmd), output=output, stderr=stderr)
print(err_msg, file=sys.stderr)
sys.exit(-1)
module_adjacency_list = None
if use_queryview:
module_adjacency_list = adjacency_list_from_queryview_xml(
module_graph, ignore_by_name, top_level_module)
else:
module_adjacency_list = adjacency_list_from_json(module_graph,
ignore_by_name,
top_level_module)
return module_adjacency_list, converted
def main():
parser = argparse.ArgumentParser(description="")
parser.add_argument("mode", help="mode: graph or report")
parser.add_argument(
"--module",
"-m",
action="append",
help="name(s) of Soong module(s). Multiple modules only supported for report"
)
parser.add_argument(
"--use_queryview",
type=bool,
default=False,
required=False,
help="whether to use queryview or module_info")
parser.add_argument(
"--ignore_by_name",
type=str,
default="",
required=False,
help="Comma-separated list. When building the tree of transitive dependencies, will not follow dependency edges pointing to module names listed by this flag."
)
args = parser.parse_args()
if len(args.module) > 1 and args.mode != "report":
print("Can only support one module with mode {}", args.mode)
mode = args.mode
use_queryview = args.use_queryview
ignore_by_name = args.ignore_by_name
report_infos = []
for top_level_module in args.module:
module_adjacency_list, converted = get_module_adjacency_list(
top_level_module, use_queryview, ignore_by_name)
if mode == "graph":
generate_dot_file(module_adjacency_list, converted, top_level_module)
elif mode == "report":
report_infos.append(
generate_report_data(module_adjacency_list, converted,
top_level_module))
else:
raise RuntimeError("unknown mode: %s" % mode)
if mode == "report":
combined_data = combine_report_data(report_infos)
generate_report(combined_data)
if __name__ == "__main__":
main()