blob: a6b86fdaa6991e02b596693b2b00ce31e7c06053 [file]
#!/usr/bin/env python3
#
# Copyright (C) 2021 The Android Open Source Project
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""A json-module-graph postprocessing script to generate a bp2build progress tracker.
Usage:
./bp2build-progress.py [report|graph] -m <module name>
Example:
To generate a report on the `adbd` module, run:
bazel run --config=bp2build --config=linux_x86_64 \
//build/bazel/scripts/bp2build-progress:bp2build-progress \
-- report -m <module-name>
To generate a graph on the `adbd` module, run:
bazel run --config=bp2build --config=linux_x86_64 \
//build/bazel/scripts/bp2build-progress:bp2build-progress \
-- graph -m adbd > /tmp/graph.in && \
dot -Tpng -o /tmp/graph.png /tmp/graph.in
"""
import argparse
import collections
import dataclasses
import datetime
import os.path
import subprocess
import sys
import xml
from typing import Dict, List, Set
import bp2build_pb2
import dependency_analysis
@dataclasses.dataclass(frozen=True, order=True)
class _ModuleInfo:
name: str
kind: str
dirname: str
num_deps: int = 0
def __str__(self):
return f"{self.name} [{self.kind}] [{self.dirname}]"
@dataclasses.dataclass(frozen=True, order=True)
class _InputModule:
module: _ModuleInfo
num_deps: int = 0
num_unconverted_deps: int = 0
def __str__(self):
total = self.num_deps
converted = self.num_deps - self.num_unconverted_deps
percent = converted / self.num_deps * 100
return f"{self.module.name}: {percent:.1f}% ({converted}/{total}) converted"
_ReportData = collections.namedtuple("_ReportData", [
"input_module",
"total_deps",
"unconverted_deps",
"all_unconverted_modules",
"blocked_modules",
"dirs_with_unconverted_modules",
"kind_of_unconverted_modules",
"converted",
])
def combine_report_data(data):
ret = _ReportData(
input_module=set(),
total_deps=set(),
unconverted_deps=set(),
all_unconverted_modules=collections.defaultdict(set),
blocked_modules=collections.defaultdict(dict),
dirs_with_unconverted_modules=set(),
kind_of_unconverted_modules=set(),
converted=set(),
)
for item in data:
ret.input_module.add(item.input_module)
ret.total_deps.update(item.total_deps)
ret.unconverted_deps.update(item.unconverted_deps)
for key, value in item.all_unconverted_modules.items():
ret.all_unconverted_modules[key].update(value)
for module, blocked in item.blocked_modules.items():
if module in ret.blocked_modules and ret.blocked_modules[
module] != blocked:
raise Exception("dependencies should match")
ret.blocked_modules[module] = blocked
ret.dirs_with_unconverted_modules.update(item.dirs_with_unconverted_modules)
ret.kind_of_unconverted_modules.update(item.kind_of_unconverted_modules)
if len(ret.converted) == 0:
ret.converted.update(item.converted)
return ret
# Generate a dot file containing the transitive closure of the module.
def generate_dot_file(modules: Dict[_ModuleInfo, Set[str]],
converted: Set[str]):
# Check that all modules in the argument are in the list of converted modules
all_converted = lambda modules: all(m in converted for m in modules)
dot_entries = []
for module, deps in modules.items():
if module.name in converted:
# Skip converted modules (nodes)
continue
dot_entries.append(
f'"{module.name}" [label="{module.name}\\n{module.kind}" color=black, style=filled, '
f"fillcolor={'yellow' if all_converted(deps) else 'tomato'}]")
dot_entries.extend(
f'"{module.name}" -> "{dep}"' for dep in deps if dep not in converted)
print("""
digraph mygraph {{
node [shape=box];
%s
}}
""" % "\n ".join(dot_entries))
# Generate a report for each module in the transitive closure, and the blockers for each module
def generate_report_data(modules, converted, input_module):
# Map of [number of unconverted deps] to list of entries,
# with each entry being the string: "<module>: <comma separated list of unconverted modules>"
blocked_modules = collections.defaultdict(set)
# Map of unconverted modules to the modules they're blocking
# (i.e. reverse deps)
all_unconverted_modules = collections.defaultdict(set)
dirs_with_unconverted_modules = set()
kind_of_unconverted_modules = set()
input_all_deps = set()
input_unconverted_deps = set()
input_module_info = None
for module, deps in sorted(modules.items()):
unconverted_deps = set(dep for dep in deps if dep not in converted)
# replace deps count with transitive deps rather than direct deps count
module = _ModuleInfo(
module.name,
module.kind,
module.dirname,
len(deps),
)
for dep in unconverted_deps:
all_unconverted_modules[dep].add(module)
unconverted_count = len(unconverted_deps)
if module.name not in converted:
if module in blocked_modules and blocked_modules[
module] != unconverted_deps:
raise Exception("dependencies should match")
blocked_modules[module] = unconverted_deps
dirs_with_unconverted_modules.add(module.dirname)
kind_of_unconverted_modules.add(module.kind)
if module.name == input_module:
input_module_info = module
input_all_deps = deps
input_unconverted_deps = unconverted_deps
return _ReportData(
input_module=_InputModule(input_module_info, len(input_all_deps),
len(input_unconverted_deps)),
total_deps=input_all_deps,
unconverted_deps=input_unconverted_deps,
all_unconverted_modules=all_unconverted_modules,
blocked_modules=blocked_modules,
dirs_with_unconverted_modules=dirs_with_unconverted_modules,
kind_of_unconverted_modules=kind_of_unconverted_modules,
converted=converted,
)
def generate_proto(report_data, file_name):
message = bp2build_pb2.Bp2buildConversionProgress(
root_modules=[m.module.name for m in report_data.input_module],
num_deps=len(report_data.total_deps),
)
for module, unconverted_deps in report_data.blocked_modules.items():
message.unconverted.add(
name=module.name,
directory=module.dirname,
type=module.kind,
unconverted_deps=unconverted_deps,
num_deps=module.num_deps,
)
with open(file_name, "wb") as f:
f.write(message.SerializeToString())
def generate_report(report_data):
report_lines = []
input_module_str = ", ".join(str(i) for i in sorted(report_data.input_module))
report_lines.append("# bp2build progress report for: %s\n" % input_module_str)
total = len(report_data.total_deps)
unconverted = len(report_data.unconverted_deps)
converted = total - unconverted
percent = converted / total * 100
report_lines.append(f"Percent converted: {percent:.2f} ({converted}/{total})")
report_lines.append(f"Total unique unconverted dependencies: {unconverted}")
report_lines.append("Ignored module types: %s\n" %
sorted(dependency_analysis.IGNORED_KINDS))
report_lines.append("# Transitive dependency closure:")
current_count = -1
for module, unconverted_deps in sorted(
report_data.blocked_modules.items(), key=lambda x: len(x[1])):
count = len(unconverted_deps)
if current_count != count:
report_lines.append(f"\n{count} unconverted deps remaining:")
current_count = count
report_lines.append("{module}: {deps}".format(
module=module, deps=", ".join(sorted(unconverted_deps))))
report_lines.append("\n")
report_lines.append("# Unconverted deps of {}:\n".format(input_module_str))
for count, dep in sorted(
((len(unconverted), dep)
for dep, unconverted in report_data.all_unconverted_modules.items()),
reverse=True):
report_lines.append("%s: blocking %d modules" % (dep, count))
report_lines.append("\n")
report_lines.append("# Dirs with unconverted modules:\n\n{}".format("\n".join(
sorted(report_data.dirs_with_unconverted_modules))))
report_lines.append("\n")
report_lines.append("# Kinds with unconverted modules:\n\n{}".format(
"\n".join(sorted(report_data.kind_of_unconverted_modules))))
report_lines.append("\n")
report_lines.append("# Converted modules:\n\n%s" %
"\n".join(sorted(report_data.converted)))
report_lines.append("\n")
report_lines.append(
"Generated by: https://cs.android.com/android/platform/superproject/+/master:build/bazel/scripts/bp2build-progress/bp2build-progress.py"
)
report_lines.append("Generated at: %s" %
datetime.datetime.now().strftime("%Y-%m-%dT%H:%M:%S %z"))
print("\n".join(report_lines))
def adjacency_list_from_json(
module_graph: ...,
ignore_by_name: List[str],
top_level_module: str,
collect_transitive_dependencies: bool = True
) -> Dict[_ModuleInfo, Set[str]]:
def filter_by_name(json):
return json["Name"] == top_level_module
module_adjacency_list = collections.defaultdict(set)
name_to_info = {}
def collect_dependencies(module, deps_names):
module_info = None
name = module["Name"]
name_to_info.setdefault(
name,
_ModuleInfo(
name=name,
kind=module["Type"],
dirname=os.path.dirname(module["Blueprint"]),
num_deps=len(deps_names),
))
module_info = name_to_info[name]
module_adjacency_list[module_info].update(deps_names)
if collect_transitive_dependencies:
for dep in deps_names:
dep_module_info = name_to_info[dep]
module_adjacency_list[module_info].update(
module_adjacency_list.get(dep_module_info, set()))
dependency_analysis.visit_json_module_graph_post_order(
module_graph, ignore_by_name, filter_by_name, collect_dependencies)
return module_adjacency_list
def adjacency_list_from_queryview_xml(
module_graph: xml.etree.ElementTree,
ignore_by_name: List[str],
top_level_module: str,
collect_transitive_dependencies: bool = True
) -> Dict[_ModuleInfo, Set[str]]:
def filter_by_name(module):
return module.name == top_level_module
module_adjacency_list = collections.defaultdict(set)
name_to_info = {}
def collect_dependencies(module, deps_names):
module_info = None
name_to_info.setdefault(
module.name,
_ModuleInfo(
name=module.name,
kind=module.kind,
dirname=module.dirname,
num_deps=len(deps_names),
))
module_info = name_to_info[module.name]
module_adjacency_list[module_info].update(deps_names)
if collect_transitive_dependencies:
for dep in deps_names:
dep_module_info = name_to_info[dep]
module_adjacency_list[module_info].update(
module_adjacency_list.get(dep_module_info, set()))
dependency_analysis.visit_queryview_xml_module_graph_post_order(
module_graph, ignore_by_name, filter_by_name, collect_dependencies)
return module_adjacency_list
def get_module_adjacency_list(
top_level_module: str,
use_queryview: bool,
ignore_by_name: List[str],
collect_transitive_dependencies: bool = True,
banchan_mode: bool = False) -> Dict[_ModuleInfo, Set[str]]:
# The main module graph containing _all_ modules in the Soong build,
# and the list of converted modules.
try:
if use_queryview:
module_graph = dependency_analysis.get_queryview_module_info(
top_level_module, banchan_mode)
module_adjacency_list = adjacency_list_from_queryview_xml(
module_graph, ignore_by_name, top_level_module,
collect_transitive_dependencies)
else:
module_graph = dependency_analysis.get_json_module_info(
top_level_module, banchan_mode)
module_adjacency_list = adjacency_list_from_json(
module_graph, ignore_by_name, top_level_module,
collect_transitive_dependencies)
except subprocess.CalledProcessError as err:
sys.exit(f"""Error running: '{' '.join(err.cmd)}':"
Stdout:
{err.stdout.decode('utf-8') if err.stdout else ''}
Stderr:
{err.stderr.decode('utf-8') if err.stderr else ''}""")
return module_adjacency_list
def main():
parser = argparse.ArgumentParser()
parser.add_argument("mode", help="mode: graph or report")
parser.add_argument(
"--module",
"-m",
action="append",
required=True,
help="name(s) of Soong module(s). Multiple modules only supported for report"
)
parser.add_argument(
"--use-queryview",
action="store_true",
help="whether to use queryview or module_info")
parser.add_argument(
"--ignore-by-name",
default="",
help="Comma-separated list. When building the tree of transitive dependencies, will not follow dependency edges pointing to module names listed by this flag."
)
parser.add_argument(
"--banchan",
action="store_true",
help="whether to run Soong in a banchan configuration rather than lunch",
)
parser.add_argument(
"--proto-file",
help="Path to write proto output",
)
args = parser.parse_args()
if len(args.module) > 1 and args.mode == "graph":
sys.exit(f"Can only support one module with mode {args.mode}")
if args.proto_file and args.mode == "graph":
sys.exit(f"Proto file only supported for report mode, not {args.mode}")
mode = args.mode
use_queryview = args.use_queryview
ignore_by_name = args.ignore_by_name.split(",")
banchan_mode = args.banchan
report_infos = []
for top_level_module in args.module:
module_adjacency_list = get_module_adjacency_list(
top_level_module,
use_queryview,
ignore_by_name,
collect_transitive_dependencies=mode != "graph",
banchan_mode=banchan_mode)
converted = dependency_analysis.get_bp2build_converted_modules()
if mode == "graph":
generate_dot_file(module_adjacency_list, converted)
elif mode == "report":
report_infos.append(
generate_report_data(module_adjacency_list, converted,
top_level_module))
else:
raise RuntimeError("unknown mode: %s" % mode)
if mode == "report":
combined_data = combine_report_data(report_infos)
generate_report(combined_data)
if args.proto_file:
generate_proto(combined_data, args.proto_file)
if __name__ == "__main__":
main()