| #!/usr/bin/env python3 |
| # Copyright (C) 2023 The Android Open Source Project |
| # |
| # Licensed under the Apache License, Version 2.0 (the "License"); |
| # you may not use this file except in compliance with the License. |
| # You may obtain a copy of the License at |
| # |
| # http://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, software |
| # distributed under the License is distributed on an "AS IS" BASIS, |
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| # See the License for the specific language governing permissions and |
| # limitations under the License. |
| # |
| """Call cargo -v, parse its output, and generate a Trusty build system module. |
| |
| Usage: Run this script in a crate workspace root directory. The Cargo.toml file |
| should work at least for the host platform. |
| |
| Without other flags, "cargo2rulesmk.py --run" calls cargo clean, calls cargo |
| build -v, and generates makefile rules. The cargo build only generates crates |
| for the host without test crates. |
| |
| If there are rustc warning messages, this script will add a warning comment to |
| the owner crate module in rules.mk. |
| """ |
| |
| import argparse |
| import glob |
| import json |
| import os |
| import os.path |
| import platform |
| import re |
| import shutil |
| import subprocess |
| import sys |
| |
| from typing import List |
| |
| |
| assert "/development/scripts" in os.path.dirname(__file__) |
| TOP_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), "../..")) |
| |
| # Some Rust packages include extra unwanted crates. |
| # This set contains all such excluded crate names. |
| EXCLUDED_CRATES = {"protobuf_bin_gen_rust_do_not_use"} |
| |
| |
| CUSTOM_MODULE_CRATES = { |
| # This map tracks Rust crates that have special modules that |
| # were not generated automatically by this script. Examples |
| # include compiler builtins and other foundational libraries. |
| # It also tracks crates tht are not under external/rust/crates. |
| "compiler_builtins": "trusty/user/base/lib/libcompiler_builtins-rust", |
| "core": "trusty/user/base/lib/libcore-rust", |
| } |
| |
| RENAME_STEM_MAP = { |
| # This map includes all changes to the default rust module stem names, |
| # which is used for output files when different from the module name. |
| "protoc_gen_rust": "protoc-gen-rust", |
| } |
| |
| # Header added to all generated rules.mk files. |
| RULES_MK_HEADER = ( |
| "# This file is generated by cargo2rulesmk.py {args}.\n" |
| + "# Do not modify this file as changes will be overridden on upgrade.\n\n" |
| ) |
| |
| CARGO_OUT = "cargo.out" # Name of file to keep cargo build -v output. |
| |
| # This should be kept in sync with tools/external_updater/crates_updater.py. |
| ERRORS_LINE = "Errors in " + CARGO_OUT + ":" |
| |
| TARGET_TMP = "target.tmp" # Name of temporary output directory. |
| |
| # Message to be displayed when this script is called without the --run flag. |
| DRY_RUN_NOTE = ( |
| "Dry-run: This script uses ./" |
| + TARGET_TMP |
| + " for output directory,\n" |
| + "runs cargo clean, runs cargo build -v, saves output to ./cargo.out,\n" |
| + "and writes to rules.mk in the current and subdirectories.\n\n" |
| + "To do do all of the above, use the --run flag.\n" |
| + "See --help for other flags, and more usage notes in this script.\n" |
| ) |
| |
| # Cargo -v output of a call to rustc. |
| RUSTC_PAT = re.compile("^ +Running `(.*\/)?rustc (.*)`$") |
| |
| # Cargo -vv output of a call to rustc could be split into multiple lines. |
| # Assume that the first line will contain some CARGO_* env definition. |
| RUSTC_VV_PAT = re.compile("^ +Running `.*CARGO_.*=.*$") |
| # The combined -vv output rustc command line pattern. |
| RUSTC_VV_CMD_ARGS = re.compile("^ *Running `.*CARGO_.*=.* (.*\/)?rustc (.*)`$") |
| |
| # Cargo -vv output of a "cc" or "ar" command; all in one line. |
| CC_AR_VV_PAT = re.compile(r'^\[([^ ]*)[^\]]*\] running:? "(cc|ar)" (.*)$') |
| # Some package, such as ring-0.13.5, has pattern '... running "cc"'. |
| |
| # Rustc output of file location path pattern for a warning message. |
| WARNING_FILE_PAT = re.compile("^ *--> ([^:]*):[0-9]+") |
| |
| # cargo test --list output of the start of running a binary. |
| CARGO_TEST_LIST_START_PAT = re.compile(r"^\s*Running (.*) \(.*\)$") |
| |
| # cargo test --list output of the end of running a binary. |
| CARGO_TEST_LIST_END_PAT = re.compile(r"^(\d+) tests?, (\d+) benchmarks$") |
| |
| CARGO2ANDROID_RUNNING_PAT = re.compile("^### Running: .*$") |
| |
| # Rust package name with suffix -d1.d2.d3(+.*)?. |
| VERSION_SUFFIX_PAT = re.compile( |
| r"^(.*)-[0-9]+\.[0-9]+\.[0-9]+(?:-(alpha|beta)\.[0-9]+)?(?:\+.*)?$" |
| ) |
| |
| # Crate types corresponding to a C ABI library |
| C_LIBRARY_CRATE_TYPES = ["staticlib", "cdylib"] |
| # Crate types corresponding to a Rust ABI library |
| RUST_LIBRARY_CRATE_TYPES = ["lib", "rlib", "dylib", "proc-macro"] |
| # Crate types corresponding to a library |
| LIBRARY_CRATE_TYPES = C_LIBRARY_CRATE_TYPES + RUST_LIBRARY_CRATE_TYPES |
| |
| |
| def altered_stem(name): |
| return RENAME_STEM_MAP[name] if (name in RENAME_STEM_MAP) else name |
| |
| |
| def is_build_crate_name(name): |
| # We added special prefix to build script crate names. |
| return name.startswith("build_script_") |
| |
| |
| def is_dependent_file_path(path): |
| # Absolute or dependent '.../' paths are not main files of this crate. |
| return path.startswith("/") or path.startswith(".../") |
| |
| |
| def get_module_name(crate): # to sort crates in a list |
| return crate.module_name |
| |
| |
| def pkg2crate_name(s): |
| return s.replace("-", "_").replace(".", "_") |
| |
| |
| def file_base_name(path): |
| return os.path.splitext(os.path.basename(path))[0] |
| |
| |
| def test_base_name(path): |
| return pkg2crate_name(file_base_name(path)) |
| |
| |
| def unquote(s): # remove quotes around str |
| if s and len(s) > 1 and s[0] == s[-1] and s[0] in ('"', "'"): |
| return s[1:-1] |
| return s |
| |
| |
| def remove_version_suffix(s): # remove -d1.d2.d3 suffix |
| if match := VERSION_SUFFIX_PAT.match(s): |
| return match.group(1) |
| return s |
| |
| |
| def short_out_name(pkg, s): # replace /.../pkg-*/out/* with .../out/* |
| return re.sub("^/.*/" + pkg + "-[0-9a-f]*/out/", ".../out/", s) |
| |
| |
| class Crate(object): |
| """Information of a Rust crate to collect/emit for a rules.mk module.""" |
| |
| def __init__(self, runner, outf_name): |
| # Remembered global runner and its members. |
| self.runner = runner |
| self.debug = runner.args.debug |
| self.cargo_dir = "" # directory of my Cargo.toml |
| self.outf_name = outf_name # path to rules.mk |
| self.outf = None # open file handle of outf_name during dump* |
| self.has_warning = False |
| # Trusty module properties derived from rustc parameters. |
| self.module_name = "" |
| self.defaults = "" # rust_defaults used by rust_test* modules |
| self.default_srcs = False # use 'srcs' defined in self.defaults |
| self.root_pkg = "" # parent package name of a sub/test packge, from -L |
| self.srcs = [] # main_src or merged multiple source files |
| self.stem = "" # real base name of output file |
| # Kept parsed status |
| self.errors = "" # all errors found during parsing |
| self.line_num = 1 # runner told input source line number |
| self.line = "" # original rustc command line parameters |
| # Parameters collected from rustc command line. |
| self.crate_name = "" # follows --crate-name |
| self.main_src = "" # follows crate_name parameter, shortened |
| self.crate_types = [] # follows --crate-type |
| self.cfgs = [] # follows --cfg, without feature= prefix |
| self.features = [] # follows --cfg, name in 'feature="..."' |
| self.codegens = [] # follows -C, some ignored |
| self.static_libs = [] # e.g. -l static=host_cpuid |
| self.shared_libs = [] # e.g. -l dylib=wayland-client, -l z |
| self.cap_lints = "" # follows --cap-lints |
| self.emit_list = "" # e.g., --emit=dep-info,metadata,link |
| self.edition = "2015" # rustc default, e.g., --edition=2018 |
| self.target = "" # follows --target |
| self.cargo_env_compat = True |
| # Parameters collected from cargo metadata output |
| self.dependencies = [] # crate dependencies output by `cargo metadata` |
| self.feature_dependencies: dict[str, List[str]] = {} # maps features to |
| # optional dependencies |
| |
| def write(self, s): |
| """convenient way to output one line at a time with EOL.""" |
| assert self.outf |
| self.outf.write(s + "\n") |
| |
| def find_cargo_dir(self): |
| """Deepest directory with Cargo.toml and contains the main_src.""" |
| if not is_dependent_file_path(self.main_src): |
| dir_name = os.path.dirname(self.main_src) |
| while dir_name: |
| if os.path.exists(dir_name + "/Cargo.toml"): |
| self.cargo_dir = dir_name |
| return |
| dir_name = os.path.dirname(dir_name) |
| |
| def add_codegens_flag(self, flag): |
| """Ignore options not used by Trusty build system""" |
| # 'prefer-dynamic' may be set by library.mk |
| # 'embed-bitcode' is ignored; we might control LTO with other flags |
| # 'codegen-units' is set globally in engine.mk |
| # 'relocation-model' and 'target-feature=+reserve-x18' may be set by |
| # common_flags.mk |
| if not ( |
| flag.startswith("codegen-units=") |
| or flag.startswith("debuginfo=") |
| or flag.startswith("embed-bitcode=") |
| or flag.startswith("extra-filename=") |
| or flag.startswith("incremental=") |
| or flag.startswith("metadata=") |
| or flag.startswith("relocation-model=") |
| or flag == "prefer-dynamic" |
| or flag == "target-feature=+reserve-x18" |
| ): |
| self.codegens.append(flag) |
| |
| def get_dependencies(self): |
| """Use output from cargo metadata to determine crate dependencies""" |
| cargo_metadata = subprocess.run( |
| [ |
| self.runner.cargo_path, |
| "metadata", |
| "--no-deps", |
| "--format-version", |
| "1", |
| ], |
| cwd=os.path.abspath(self.cargo_dir), |
| stdout=subprocess.PIPE, |
| check=False, |
| ) |
| if cargo_metadata.returncode: |
| self.errors += ( |
| "ERROR: unable to get cargo metadata to determine " |
| f"dependencies; return code {cargo_metadata.returncode}\n" |
| ) |
| else: |
| metadata_json = json.loads(cargo_metadata.stdout) |
| |
| for package in metadata_json["packages"]: |
| # package names containing '-' are changed to '_' in crate_name |
| if package["name"].replace("-", "_") == self.crate_name: |
| self.dependencies = package["dependencies"] |
| for feat, props in package["features"].items(): |
| feat_deps = [ |
| d[4:] for d in props if d.startswith("dep:") |
| ] |
| if feat_deps and feat in self.feature_dependencies: |
| self.feature_dependencies[feat].extend(feat_deps) |
| else: |
| self.feature_dependencies[feat] = feat_deps |
| break |
| else: # package name not found in metadata |
| if is_build_crate_name(self.crate_name): |
| print( |
| "### WARNING: unable to determine dependencies for " |
| + f"{self.crate_name} from cargo metadata" |
| ) |
| |
| def parse(self, line_num, line): |
| """Find important rustc arguments to convert to makefile rules.""" |
| self.line_num = line_num |
| self.line = line |
| args = [unquote(l) for l in line.split()] |
| i = 0 |
| # Loop through every argument of rustc. |
| while i < len(args): |
| arg = args[i] |
| if arg == "--crate-name": |
| i += 1 |
| self.crate_name = args[i] |
| elif arg == "--crate-type": |
| i += 1 |
| # cargo calls rustc with multiple --crate-type flags. |
| # rustc can accept: |
| # --crate-type [bin|lib|rlib|dylib|cdylib|staticlib|proc-macro] |
| self.crate_types.append(args[i]) |
| elif arg == "--test": |
| self.crate_types.append("test") |
| elif arg == "--target": |
| i += 1 |
| self.target = args[i] |
| elif arg == "--cfg": |
| i += 1 |
| if args[i].startswith("feature="): |
| self.features.append( |
| unquote(args[i].replace("feature=", "")) |
| ) |
| else: |
| self.cfgs.append(args[i]) |
| elif arg == "--extern": |
| i += 1 |
| pass # ignored; get all dependencies from cargo metadata |
| elif arg == "-C": # codegen options |
| i += 1 |
| self.add_codegens_flag(args[i]) |
| elif arg.startswith("-C"): |
| # cargo has been passing "-C <xyz>" flag to rustc, |
| # but newer cargo could pass '-Cembed-bitcode=no' to rustc. |
| self.add_codegens_flag(arg[2:]) |
| elif arg == "--cap-lints": |
| i += 1 |
| self.cap_lints = args[i] |
| elif arg == "-L": |
| i += 1 |
| if args[i].startswith("dependency=") and args[i].endswith( |
| "/deps" |
| ): |
| if "/" + TARGET_TMP + "/" in args[i]: |
| self.root_pkg = re.sub( |
| "^.*/", |
| "", |
| re.sub("/" + TARGET_TMP + "/.*/deps$", "", args[i]), |
| ) |
| else: |
| self.root_pkg = re.sub( |
| "^.*/", |
| "", |
| re.sub("/[^/]+/[^/]+/deps$", "", args[i]), |
| ) |
| self.root_pkg = remove_version_suffix(self.root_pkg) |
| elif arg == "-l": |
| i += 1 |
| if args[i].startswith("static="): |
| self.static_libs.append(re.sub("static=", "", args[i])) |
| elif args[i].startswith("dylib="): |
| self.shared_libs.append(re.sub("dylib=", "", args[i])) |
| else: |
| self.shared_libs.append(args[i]) |
| elif arg in ("--out-dir", "--color"): # ignored |
| i += 1 |
| elif arg.startswith("--error-format=") or arg.startswith("--json="): |
| pass # ignored |
| elif arg.startswith("--emit="): |
| self.emit_list = arg.replace("--emit=", "") |
| elif arg.startswith("--edition="): |
| self.edition = arg.replace("--edition=", "") |
| elif arg.startswith("-Aclippy") or arg.startswith("-Wclippy"): |
| pass # TODO: emit these flags in rules.mk |
| elif arg.startswith("-W"): |
| pass # ignored |
| elif arg.startswith("-Z"): |
| pass # ignore unstable flags |
| elif arg.startswith("-D"): |
| pass # TODO: emit these flags in rules.mk |
| elif not arg.startswith("-"): |
| # shorten imported crate main source paths like $HOME/.cargo/ |
| # registry/src/github.com-1ecc6299db9ec823/memchr-2.3.3/src/ |
| # lib.rs |
| self.main_src = re.sub( |
| r"^/[^ ]*/registry/src/", ".../", args[i] |
| ) |
| self.main_src = re.sub( |
| r"^\.\.\./github.com-[0-9a-f]*/", ".../", self.main_src |
| ) |
| self.find_cargo_dir() |
| if self.cargo_dir: # for a subdirectory |
| if ( |
| self.runner.args.no_subdir |
| ): # all .mk content to /dev/null |
| self.outf_name = "/dev/null" |
| elif not self.runner.args.onefile: |
| # Write to rules.mk in the subdirectory with Cargo.toml. |
| self.outf_name = self.cargo_dir + "/rules.mk" |
| self.main_src = self.main_src[len(self.cargo_dir) + 1 :] |
| |
| else: |
| self.errors += "ERROR: unknown " + arg + "\n" |
| i += 1 |
| if not self.crate_name: |
| self.errors += "ERROR: missing --crate-name\n" |
| if not self.main_src: |
| self.errors += "ERROR: missing main source file\n" |
| else: |
| self.srcs.append(self.main_src) |
| if not self.crate_types: |
| # Treat "--cfg test" as "--test" |
| if "test" in self.cfgs: |
| self.crate_types.append("test") |
| else: |
| self.errors += "ERROR: missing --crate-type or --test\n" |
| elif len(self.crate_types) > 1: |
| if "test" in self.crate_types: |
| self.errors += ( |
| "ERROR: cannot handle both --crate-type and --test\n" |
| ) |
| if "lib" in self.crate_types and "rlib" in self.crate_types: |
| self.errors += ( |
| "ERROR: cannot generate both lib and rlib crate types\n" |
| ) |
| if not self.root_pkg: |
| self.root_pkg = self.crate_name |
| |
| # get the package dependencies by running cargo metadata |
| if not self.skip_crate(): |
| self.get_dependencies() |
| self.cfgs = sorted(set(self.cfgs)) |
| self.features = sorted(set(self.features)) |
| self.codegens = sorted(set(self.codegens)) |
| self.static_libs = sorted(set(self.static_libs)) |
| self.shared_libs = sorted(set(self.shared_libs)) |
| self.crate_types = sorted(set(self.crate_types)) |
| self.module_name = self.stem |
| return self |
| |
| def dump_line(self): |
| self.write("\n// Line " + str(self.line_num) + " " + self.line) |
| |
| def feature_list(self): |
| """Return a string of main_src + "feature_list".""" |
| pkg = self.main_src |
| if pkg.startswith(".../"): # keep only the main package name |
| pkg = re.sub("/.*", "", pkg[4:]) |
| elif pkg.startswith("/"): # use relative path for a local package |
| pkg = os.path.relpath(pkg) |
| if not self.features: |
| return pkg |
| return pkg + ' "' + ",".join(self.features) + '"' |
| |
| def dump_skip_crate(self, kind): |
| if self.debug: |
| self.write("\n// IGNORED: " + kind + " " + self.main_src) |
| return self |
| |
| def skip_crate(self): |
| """Return crate_name or a message if this crate should be skipped.""" |
| if ( |
| is_build_crate_name(self.crate_name) |
| or self.crate_name in EXCLUDED_CRATES |
| ): |
| return self.crate_name |
| if is_dependent_file_path(self.main_src): |
| return "dependent crate" |
| return "" |
| |
| def dump(self): |
| """Dump all error/debug/module code to the output rules.mk file.""" |
| self.runner.init_rules_file(self.outf_name) |
| with open(self.outf_name, "a", encoding="utf-8") as outf: |
| self.outf = outf |
| if self.errors: |
| self.dump_line() |
| self.write(self.errors) |
| elif self.skip_crate(): |
| self.dump_skip_crate(self.skip_crate()) |
| else: |
| if self.debug: |
| self.dump_debug_info() |
| self.dump_trusty_module() |
| self.outf = None |
| |
| def dump_debug_info(self): |
| """Dump parsed data, when cargo2rulesmk is called with --debug.""" |
| |
| def dump(name, value): |
| self.write(f"//{name:>12} = {value}") |
| |
| def opt_dump(name, value): |
| if value: |
| dump(name, value) |
| |
| def dump_list(fmt, values): |
| for v in values: |
| self.write(fmt % v) |
| |
| self.dump_line() |
| dump("module_name", self.module_name) |
| dump("crate_name", self.crate_name) |
| dump("crate_types", self.crate_types) |
| dump("main_src", self.main_src) |
| dump("has_warning", self.has_warning) |
| opt_dump("target", self.target) |
| opt_dump("edition", self.edition) |
| opt_dump("emit_list", self.emit_list) |
| opt_dump("cap_lints", self.cap_lints) |
| dump_list("// cfg = %s", self.cfgs) |
| dump_list("// cfg = 'feature \"%s\"'", self.features) |
| # TODO(chh): escape quotes in self.features, but not in other dump_list |
| dump_list("// codegen = %s", self.codegens) |
| dump_list("// -l static = %s", self.static_libs) |
| dump_list("// -l (dylib) = %s", self.shared_libs) |
| |
| def dump_trusty_module(self): |
| """Dump one or more module definitions, depending on crate_types.""" |
| if len(self.crate_types) > 1: |
| if "test" in self.crate_types: |
| self.write("\nERROR: multiple crate types cannot include test type") |
| return |
| |
| if "lib" in self.crate_types: |
| print(f"### WARNING: crate {self.crate_name} has multiple " |
| f"crate types ({str(self.crate_types)}). Treating as 'lib'") |
| self.crate_types = ["lib"] |
| else: |
| self.write("\nERROR: don't know how to handle crate types of " |
| f"crate {self.crate_name}: {str(self.crate_types)}") |
| return |
| |
| self.dump_single_type_trusty_module() |
| |
| def dump_srcs_list(self): |
| """Dump the srcs list, for defaults or regular modules.""" |
| if len(self.srcs) > 1: |
| srcs = sorted(set(self.srcs)) # make a copy and dedup |
| else: |
| srcs = [self.main_src] |
| self.write("MODULE_SRCS := \\") |
| for src in srcs: |
| self.write(f"\t$(LOCAL_DIR)/{src} \\") |
| self.write("") |
| |
| # add rust file generated by build.rs to MODULE_SRCDEPS, if any |
| # TODO(perlarsen): is there a need to support more than one output file? |
| if srcdeps := [ |
| f for f in self.runner.build_out_files if f.endswith(".rs") |
| ]: |
| assert len(srcdeps) == 1 |
| outfile = srcdeps.pop() |
| lines = [ |
| f"OUT_FILE := $(call TOBUILDDIR,$(MODULE))/{outfile}", |
| f"$(OUT_FILE): $(MODULE)/out/{outfile}", |
| "\t@echo copying $< to $@", |
| "\t@$(MKDIR)", |
| "\tcp $< $@", |
| "", |
| "MODULE_RUST_ENV += OUT_DIR=$(dir $(OUT_FILE))", |
| "", |
| "MODULE_SRCDEPS := $(OUT_FILE)", |
| ] |
| self.write("\n".join(lines)) |
| |
| def dump_single_type_trusty_module(self): |
| """Dump one simple Trusty module, which has only one crate_type.""" |
| crate_type = self.crate_types[0] |
| assert crate_type != "test" |
| self.dump_one_trusty_module(crate_type) |
| |
| def dump_one_trusty_module(self, crate_type): |
| """Dump one Trusty module definition.""" |
| if crate_type in ["test", "bin"]: # TODO: support test crates |
| print( |
| f"### WARNING: ignoring {crate_type} crate: {self.crate_name}") |
| return |
| if self.codegens: # TODO: support crates that require codegen flags |
| print( |
| f"ERROR: {self.crate_name} uses unexpected codegen flags: " + |
| str(self.codegens) |
| ) |
| return |
| |
| self.dump_core_properties() |
| if not self.defaults: |
| self.dump_edition_flags_libs() |
| |
| # NOTE: a crate may list the same dependency as required and optional |
| library_deps = set() |
| for dependency in self.dependencies: |
| if dependency["kind"] in ["dev", "build"]: |
| continue |
| name = ( |
| rename |
| if (rename := dependency["rename"]) |
| else dependency["name"] |
| ) |
| if dependency["target"]: |
| print( |
| f"### WARNING: ignoring target-specific dependency: {name}") |
| continue |
| path = CUSTOM_MODULE_CRATES.get( |
| name, f"external/rust/crates/{name}" |
| ) |
| if dependency["optional"]: |
| if not any( |
| name in self.feature_dependencies.get(f, []) |
| for f in self.features |
| ): |
| continue |
| library_deps.add(path) |
| if library_deps: |
| self.write("MODULE_LIBRARY_DEPS := \\") |
| for path in sorted(library_deps): |
| self.write(f"\t{path} \\") |
| self.write("") |
| if crate_type == "test" and not self.default_srcs: |
| raise NotImplementedError("Crates with test data are not supported") |
| |
| assert crate_type in LIBRARY_CRATE_TYPES |
| self.write("include make/library.mk") |
| |
| def dump_edition_flags_libs(self): |
| if self.edition: |
| self.write(f"MODULE_RUST_EDITION := {self.edition}") |
| if self.features or self.cfgs: |
| self.write("MODULE_RUSTFLAGS += \\") |
| for feature in self.features: |
| self.write(f"\t--cfg 'feature=\"{feature}\"' \\") |
| for cfg in self.cfgs: |
| self.write(f"\t--cfg '{cfg}' \\") |
| self.write("") |
| |
| if self.static_libs or self.shared_libs: |
| print("### WARNING: Crates with depend on static or shared " |
| "libraries are not supported") |
| |
| def main_src_basename_path(self): |
| return re.sub("/", "_", re.sub(".rs$", "", self.main_src)) |
| |
| def test_module_name(self): |
| """Return a unique name for a test module.""" |
| # root_pkg+(_host|_device) + '_test_'+source_file_name |
| suffix = self.main_src_basename_path() |
| return self.root_pkg + "_test_" + suffix |
| |
| def dump_core_properties(self): |
| """Dump the module header, name, stem, etc.""" |
| self.write("LOCAL_DIR := $(GET_LOCAL_DIR)") |
| self.write("MODULE := $(LOCAL_DIR)") |
| self.write(f"MODULE_CRATE_NAME := {self.crate_name}") |
| |
| # Trusty's module system only supports bin, rlib, and proc-macro so map |
| # lib->rlib |
| if self.crate_types != ["lib"]: |
| crate_types = set( |
| "rlib" if ct == "lib" else ct for ct in self.crate_types |
| ) |
| self.write(f'MODULE_RUST_CRATE_TYPES := {" ".join(crate_types)}') |
| |
| if not self.default_srcs: |
| self.dump_srcs_list() |
| |
| if hasattr(self.runner.args, "module_add_implicit_deps"): |
| if hasattr(self.runner.args, "module_add_implicit_deps_reason"): |
| self.write(self.runner.args.module_add_implicit_deps_reason) |
| |
| if self.runner.args.module_add_implicit_deps in [True, "yes"]: |
| self.write("MODULE_ADD_IMPLICIT_DEPS := true") |
| elif self.runner.args.module_add_implicit_deps in [False, "no"]: |
| self.write("MODULE_ADD_IMPLICIT_DEPS := false") |
| else: |
| sys.exit( |
| "ERROR: invalid value for module_add_implicit_deps: " + |
| str(self.runner.args.module_add_implicit_deps) |
| ) |
| |
| |
| class Runner(object): |
| """Main class to parse cargo -v output and print Trusty makefile modules.""" |
| |
| def __init__(self, args): |
| self.mk_files = set() # Remember all Trusty module files. |
| self.root_pkg = "" # name of package in ./Cargo.toml |
| # Saved flags, modes, and data. |
| self.args = args |
| self.dry_run = not args.run |
| self.skip_cargo = args.skipcargo |
| self.cargo_path = "./cargo" # path to cargo, will be set later |
| self.checked_out_files = False # to check only once |
| self.build_out_files = [] # output files generated by build.rs |
| self.crates: List[Crate] = [] |
| self.warning_files = set() |
| # Keep a unique mapping from (module name) to crate |
| self.name_owners = {} |
| # Save and dump all errors from cargo to rules.mk. |
| self.errors = "" |
| self.test_errors = "" |
| self.setup_cargo_path() |
| # Default action is cargo clean, followed by build or user given actions |
| if args.cargo: |
| self.cargo = ["clean"] + args.cargo |
| else: |
| default_target = "--target x86_64-unknown-linux-gnu" |
| # Use the same target for both host and default device builds. |
| # Same target is used as default in host x86_64 Android compilation. |
| # Note: b/169872957, prebuilt cargo failed to build vsock |
| # on x86_64-unknown-linux-musl systems. |
| self.cargo = ["clean", "build " + default_target] |
| if args.tests: |
| self.cargo.append("build --tests " + default_target) |
| self.empty_tests = set() |
| self.empty_unittests = False |
| |
| def setup_cargo_path(self): |
| """Find cargo in the --cargo_bin or prebuilt rust bin directory.""" |
| if self.args.cargo_bin: |
| self.cargo_path = os.path.join(self.args.cargo_bin, "cargo") |
| if not os.path.isfile(self.cargo_path): |
| sys.exit("ERROR: cannot find cargo in " + self.args.cargo_bin) |
| print("INFO: using cargo in " + self.args.cargo_bin) |
| return |
| # TODO(perlarsen): try getting cargo from $RUST_BINDIR set in envsetup.sh |
| # We have only tested this on Linux. |
| if platform.system() != "Linux": |
| sys.exit( |
| "ERROR: this script has only been tested on Linux with cargo." |
| ) |
| # Assuming that this script is in development/scripts |
| linux_dir = os.path.join( |
| TOP_DIR, "prebuilts", "rust", "linux-x86" |
| ) |
| if not os.path.isdir(linux_dir): |
| sys.exit("ERROR: cannot find directory " + linux_dir) |
| rust_version = self.find_rust_version(linux_dir) |
| if self.args.verbose: |
| print(f"### INFO: using prebuilt rust version {rust_version}") |
| cargo_bin = os.path.join(linux_dir, rust_version, "bin") |
| self.cargo_path = os.path.join(cargo_bin, "cargo") |
| if not os.path.isfile(self.cargo_path): |
| sys.exit( |
| "ERROR: cannot find cargo in " |
| + cargo_bin |
| + "; please try --cargo_bin= flag." |
| ) |
| return |
| |
| def find_rust_version(self, linux_dir): |
| """find newest prebuilt rust version.""" |
| # find the newest (largest) version number in linux_dir. |
| rust_version = (0, 0, 0) # the prebuilt version to use |
| version_pat = re.compile(r"([0-9]+)\.([0-9]+)\.([0-9]+)$") |
| for dir_name in os.listdir(linux_dir): |
| result = version_pat.match(dir_name) |
| if not result: |
| continue |
| version = ( |
| int(result.group(1)), |
| int(result.group(2)), |
| int(result.group(3)), |
| ) |
| if version > rust_version: |
| rust_version = version |
| return ".".join(str(ver) for ver in rust_version) |
| |
| def find_out_files(self): |
| # list1 has build.rs output for normal crates |
| list1 = glob.glob( |
| TARGET_TMP + "/*/*/build/" + self.root_pkg + "-*/out/*" |
| ) |
| # list2 has build.rs output for proc-macro crates |
| list2 = glob.glob(TARGET_TMP + "/*/build/" + self.root_pkg + "-*/out/*") |
| return list1 + list2 |
| |
| def copy_out_files(self): |
| """Copy build.rs output files to ./out and set up build_out_files.""" |
| if self.checked_out_files: |
| return |
| self.checked_out_files = True |
| cargo_out_files = self.find_out_files() |
| out_files = set() |
| if cargo_out_files: |
| os.makedirs("out", exist_ok=True) |
| for path in cargo_out_files: |
| file_name = path.split("/")[-1] |
| out_files.add(file_name) |
| shutil.copy(path, "out/" + file_name) |
| self.build_out_files = sorted(out_files) |
| |
| def has_used_out_dir(self): |
| """Returns true if env!("OUT_DIR") is found.""" |
| return 0 == os.system( |
| "grep -rl --exclude build.rs --include \\*.rs" |
| + " 'env!(\"OUT_DIR\")' * > /dev/null" |
| ) |
| |
| def init_rules_file(self, name): |
| # name could be rules.mk or sub_dir_path/rules.mk |
| if name not in self.mk_files: |
| self.mk_files.add(name) |
| with open(name, "w", encoding="utf-8") as outf: |
| print_args = sys.argv[1:].copy() |
| if "--cargo_bin" in print_args: |
| index = print_args.index("--cargo_bin") |
| del print_args[index : index + 2] |
| outf.write(RULES_MK_HEADER.format(args=" ".join(print_args))) |
| |
| def find_root_pkg(self): |
| """Read name of [package] in ./Cargo.toml.""" |
| if not os.path.exists("./Cargo.toml"): |
| return |
| with open("./Cargo.toml", "r", encoding="utf-8") as inf: |
| pkg_section = re.compile(r"^ *\[package\]") |
| name = re.compile('^ *name *= * "([^"]*)"') |
| in_pkg = False |
| for line in inf: |
| if in_pkg: |
| if match := name.match(line): |
| self.root_pkg = match.group(1) |
| break |
| else: |
| in_pkg = pkg_section.match(line) is not None |
| |
| def run_cargo(self): |
| """Calls cargo -v and save its output to ./cargo.out.""" |
| if self.skip_cargo: |
| return self |
| cargo_toml = "./Cargo.toml" |
| cargo_out = "./cargo.out" |
| |
| # Do not use Cargo.lock, because Trusty makefile rules are designed |
| # to run with the latest available vendored crates in Trusty. |
| cargo_lock = "./Cargo.lock" |
| cargo_lock_saved = "./cargo.lock.saved" |
| had_cargo_lock = os.path.exists(cargo_lock) |
| if not os.access(cargo_toml, os.R_OK): |
| print("ERROR: Cannot find or read", cargo_toml) |
| return self |
| if not self.dry_run: |
| if os.path.exists(cargo_out): |
| os.remove(cargo_out) |
| if not self.args.use_cargo_lock and had_cargo_lock: # save it |
| os.rename(cargo_lock, cargo_lock_saved) |
| cmd_tail_target = " --target-dir " + TARGET_TMP |
| cmd_tail_redir = " >> " + cargo_out + " 2>&1" |
| # set up search PATH for cargo to find the correct rustc |
| saved_path = os.environ["PATH"] |
| os.environ["PATH"] = os.path.dirname(self.cargo_path) + ":" + saved_path |
| # Add [workspace] to Cargo.toml if it is not there. |
| added_workspace = False |
| cargo_toml_lines = None |
| if self.args.add_workspace: |
| with open(cargo_toml, "r", encoding="utf-8") as in_file: |
| cargo_toml_lines = in_file.readlines() |
| found_workspace = "[workspace]\n" in cargo_toml_lines |
| if found_workspace: |
| print("### WARNING: found [workspace] in Cargo.toml") |
| else: |
| with open(cargo_toml, "a", encoding="utf-8") as out_file: |
| out_file.write("\n\n[workspace]\n") |
| added_workspace = True |
| if self.args.verbose: |
| print("### INFO: added [workspace] to Cargo.toml") |
| features = "" |
| for c in self.cargo: |
| features = "" |
| if c != "clean": |
| if self.args.features is not None: |
| features = " --no-default-features" |
| if self.args.features: |
| features += " --features " + self.args.features |
| cmd_v_flag = " -vv " if self.args.vv else " -v " |
| cmd = self.cargo_path + cmd_v_flag |
| cmd += c + features + cmd_tail_target + cmd_tail_redir |
| if c != "clean": |
| rustflags = self.args.rustflags if self.args.rustflags else "" |
| # linting issues shouldn't prevent us from generating rules.mk |
| rustflags = f'RUSTFLAGS="{rustflags} --cap-lints allow" ' |
| cmd = rustflags + cmd |
| self.run_cmd(cmd, cargo_out) |
| if self.args.tests: |
| cmd = ( |
| self.cargo_path |
| + " test" |
| + features |
| + cmd_tail_target |
| + " -- --list" |
| + cmd_tail_redir |
| ) |
| self.run_cmd(cmd, cargo_out) |
| if added_workspace: # restore original Cargo.toml |
| with open(cargo_toml, "w", encoding="utf-8") as out_file: |
| assert cargo_toml_lines |
| out_file.writelines(cargo_toml_lines) |
| if self.args.verbose: |
| print("### INFO: restored original Cargo.toml") |
| os.environ["PATH"] = saved_path |
| if not self.dry_run: |
| if not had_cargo_lock: # restore to no Cargo.lock state |
| if os.path.exists(cargo_lock): |
| os.remove(cargo_lock) |
| elif not self.args.use_cargo_lock: # restore saved Cargo.lock |
| os.rename(cargo_lock_saved, cargo_lock) |
| return self |
| |
| def run_cmd(self, cmd, cargo_out): |
| if self.dry_run: |
| print("Dry-run skip:", cmd) |
| else: |
| if self.args.verbose: |
| print("Running:", cmd) |
| with open(cargo_out, "a+", encoding="utf-8") as out_file: |
| out_file.write("### Running: " + cmd + "\n") |
| ret = os.system(cmd) |
| if ret != 0: |
| print( |
| "*** There was an error while running cargo. " |
| + f"See the {cargo_out} file for details." |
| ) |
| |
| def apply_patch(self): |
| """Apply local patch file if it is given.""" |
| if self.args.patch: |
| if self.dry_run: |
| print("Dry-run skip patch file:", self.args.patch) |
| else: |
| if not os.path.exists(self.args.patch): |
| self.append_to_rules( |
| "ERROR cannot find patch file: " + self.args.patch |
| ) |
| return self |
| if self.args.verbose: |
| print( |
| "### INFO: applying local patch file:", self.args.patch |
| ) |
| subprocess.run( |
| [ |
| "patch", |
| "-s", |
| "--no-backup-if-mismatch", |
| "./rules.mk", |
| self.args.patch, |
| ], |
| check=True, |
| ) |
| return self |
| |
| def gen_rules(self): |
| """Parse cargo.out and generate Trusty makefile rules""" |
| if self.dry_run: |
| print("Dry-run skip: read", CARGO_OUT, "write rules.mk") |
| elif os.path.exists(CARGO_OUT): |
| self.find_root_pkg() |
| if self.args.copy_out: |
| self.copy_out_files() |
| elif self.find_out_files() and self.has_used_out_dir(): |
| print( |
| "WARNING: " |
| + self.root_pkg |
| + " has cargo output files; " |
| + "please rerun with the --copy-out flag." |
| ) |
| with open(CARGO_OUT, "r", encoding="utf-8") as cargo_out: |
| self.parse(cargo_out, "rules.mk") |
| self.crates.sort(key=get_module_name) |
| for crate in self.crates: |
| crate.dump() |
| if self.errors: |
| self.append_to_rules("\n" + ERRORS_LINE + "\n" + self.errors) |
| if self.test_errors: |
| self.append_to_rules( |
| "\n// Errors when listing tests:\n" + self.test_errors |
| ) |
| return self |
| |
| def add_crate(self, crate: Crate): |
| """Append crate to list unless it meets criteria for being skipped.""" |
| if crate.skip_crate(): |
| if self.args.debug: # include debug info of all crates |
| self.crates.append(crate) |
| elif crate.crate_types == set(["bin"]): |
| print("WARNING: skipping binary crate: " + crate.crate_name) |
| else: |
| self.crates.append(crate) |
| |
| def find_warning_owners(self): |
| """For each warning file, find its owner crate.""" |
| missing_owner = False |
| for f in self.warning_files: |
| cargo_dir = "" # find lowest crate, with longest path |
| owner = None # owner crate of this warning |
| for c in self.crates: |
| if f.startswith(c.cargo_dir + "/") and len(cargo_dir) < len( |
| c.cargo_dir |
| ): |
| cargo_dir = c.cargo_dir |
| owner = c |
| if owner: |
| owner.has_warning = True |
| else: |
| missing_owner = True |
| if missing_owner and os.path.exists("Cargo.toml"): |
| # owner is the root cargo, with empty cargo_dir |
| for c in self.crates: |
| if not c.cargo_dir: |
| c.has_warning = True |
| |
| def rustc_command(self, n, rustc_line, line, outf_name): |
| """Process a rustc command line from cargo -vv output.""" |
| # cargo build -vv output can have multiple lines for a rustc command |
| # due to '\n' in strings for environment variables. |
| # strip removes leading spaces and '\n' at the end |
| new_rustc = (rustc_line.strip() + line) if rustc_line else line |
| # Use an heuristic to detect the completions of a multi-line command. |
| # This might fail for some very rare case, but easy to fix manually. |
| if not line.endswith("`\n") or (new_rustc.count("`") % 2) != 0: |
| return new_rustc |
| if match := RUSTC_VV_CMD_ARGS.match(new_rustc): |
| args = match.group(2) |
| self.add_crate(Crate(self, outf_name).parse(n, args)) |
| else: |
| self.assert_empty_vv_line(new_rustc) |
| return "" |
| |
| def append_to_rules(self, line): |
| self.init_rules_file("rules.mk") |
| with open("rules.mk", "a", encoding="utf-8") as outf: |
| outf.write(line) |
| |
| def assert_empty_vv_line(self, line): |
| if line: # report error if line is not empty |
| self.append_to_rules("ERROR -vv line: " + line) |
| return "" |
| |
| def add_empty_test(self, name): |
| if name.startswith("unittests"): |
| self.empty_unittests = True |
| else: |
| self.empty_tests.add(name) |
| |
| def should_ignore_test(self, src): |
| # cargo test outputs the source file for integration tests but |
| # "unittests" for unit tests. To figure out to which crate this |
| # corresponds, we check if the current source file is the main source of |
| # a non-test crate, e.g., a library or a binary. |
| return ( |
| src in self.args.test_blocklist |
| or src in self.empty_tests |
| or ( |
| self.empty_unittests |
| and src |
| in [ |
| c.main_src for c in self.crates if c.crate_types != ["test"] |
| ] |
| ) |
| ) |
| |
| def parse(self, inf, outf_name): |
| """Parse rustc, test, and warning messages in input file.""" |
| n = 0 # line number |
| # We read the file in two passes, where the first simply checks for |
| # empty tests. Otherwise we would add and merge tests before seeing |
| # they're empty. |
| cur_test_name = None |
| for line in inf: |
| if match := CARGO_TEST_LIST_START_PAT.match(line): |
| cur_test_name = match.group(1) |
| elif cur_test_name and ( |
| match := CARGO_TEST_LIST_END_PAT.match(line) |
| ): |
| if int(match.group(1)) + int(match.group(2)) == 0: |
| self.add_empty_test(cur_test_name) |
| cur_test_name = None |
| inf.seek(0) |
| prev_warning = False # true if the previous line was warning: ... |
| rustc_line = "" # previous line(s) matching RUSTC_VV_PAT |
| in_tests = False |
| for line in inf: |
| n += 1 |
| if line.startswith("warning: "): |
| prev_warning = True |
| rustc_line = self.assert_empty_vv_line(rustc_line) |
| continue |
| new_rustc = "" |
| if match := RUSTC_PAT.match(line): |
| args_line = match.group(2) |
| self.add_crate(Crate(self, outf_name).parse(n, args_line)) |
| self.assert_empty_vv_line(rustc_line) |
| elif rustc_line or RUSTC_VV_PAT.match(line): |
| new_rustc = self.rustc_command(n, rustc_line, line, outf_name) |
| elif CC_AR_VV_PAT.match(line): |
| raise NotImplementedError("$CC or $AR commands not supported") |
| elif prev_warning and (match := WARNING_FILE_PAT.match(line)): |
| self.assert_empty_vv_line(rustc_line) |
| fpath = match.group(1) |
| if fpath[0] != "/": # ignore absolute path |
| self.warning_files.add(fpath) |
| elif line.startswith("error: ") or line.startswith("error[E"): |
| if not self.args.ignore_cargo_errors: |
| if in_tests: |
| self.test_errors += "// " + line |
| else: |
| self.errors += line |
| elif CARGO2ANDROID_RUNNING_PAT.match(line): |
| in_tests = "cargo test" in line and "--list" in line |
| prev_warning = False |
| rustc_line = new_rustc |
| self.find_warning_owners() |
| |
| |
| def get_parser(): |
| """Parse main arguments.""" |
| parser = argparse.ArgumentParser("cargo2rulesmk") |
| parser.add_argument( |
| "--add_workspace", |
| action="store_true", |
| default=False, |
| help=( |
| "append [workspace] to Cargo.toml before calling cargo," |
| + " to treat current directory as root of package source;" |
| + " otherwise the relative source file path in generated" |
| + " rules.mk file will be from the parent directory." |
| ), |
| ) |
| parser.add_argument( |
| "--cargo", |
| action="append", |
| metavar="args_string", |
| help=( |
| "extra cargo build -v args in a string, " |
| + "each --cargo flag calls cargo build -v once" |
| ), |
| ) |
| parser.add_argument( |
| "--cargo_bin", |
| type=str, |
| help="use cargo in the cargo_bin directory instead of the prebuilt one", |
| ) |
| parser.add_argument( |
| "--copy-out", |
| action="store_true", |
| default=False, |
| help=( |
| "only for root directory, " |
| + "copy build.rs output to ./out/* and declare source deps " |
| + "for ./out/*.rs; for crates with code pattern: " |
| + 'include!(concat!(env!("OUT_DIR"), "/<some_file>.rs"))' |
| ), |
| ) |
| parser.add_argument( |
| "--debug", |
| action="store_true", |
| default=False, |
| help="dump debug info into rules.mk", |
| ) |
| parser.add_argument( |
| "--features", |
| type=str, |
| help=( |
| "pass features to cargo build, " |
| + "empty string means no default features" |
| ), |
| ) |
| parser.add_argument( |
| "--ignore-cargo-errors", |
| action="store_true", |
| default=False, |
| help="do not append cargo/rustc error messages to rules.mk", |
| ) |
| parser.add_argument( |
| "--no-subdir", |
| action="store_true", |
| default=False, |
| help="do not output anything for sub-directories", |
| ) |
| parser.add_argument( |
| "--onefile", |
| action="store_true", |
| default=False, |
| help=( |
| "output all into one ./rules.mk, default will generate " |
| + "one rules.mk per Cargo.toml in subdirectories" |
| ), |
| ) |
| parser.add_argument( |
| "--patch", |
| type=str, |
| help="apply the given patch file to generated ./rules.mk", |
| ) |
| parser.add_argument( |
| "--run", |
| action="store_true", |
| default=False, |
| help="run it, default is dry-run", |
| ) |
| parser.add_argument("--rustflags", type=str, help="passing flags to rustc") |
| parser.add_argument( |
| "--skipcargo", |
| action="store_true", |
| default=False, |
| help="skip cargo command, parse cargo.out, and generate ./rules.mk", |
| ) |
| parser.add_argument( |
| "--tests", |
| action="store_true", |
| default=False, |
| help="run cargo build --tests after normal build", |
| ) |
| parser.add_argument( |
| "--use-cargo-lock", |
| action="store_true", |
| default=False, |
| help=( |
| "run cargo build with existing Cargo.lock " |
| + "(used when some latest dependent crates failed)" |
| ), |
| ) |
| parser.add_argument( |
| "--test-data", |
| nargs="*", |
| default=[], |
| help=( |
| "Add the given file to the given test's data property. " |
| + "Usage: test-path=data-path" |
| ), |
| ) |
| parser.add_argument( |
| "--dependency-blocklist", |
| nargs="*", |
| default=[], |
| help="Do not emit the given dependencies (without lib prefixes).", |
| ) |
| parser.add_argument( |
| "--test-blocklist", |
| nargs="*", |
| default=[], |
| help=( |
| "Do not emit the given tests. " |
| + "Pass the path to the test file to exclude." |
| ), |
| ) |
| parser.add_argument( |
| "--cfg-blocklist", |
| nargs="*", |
| default=[], |
| help="Do not emit the given cfg.", |
| ) |
| parser.add_argument( |
| "--verbose", |
| action="store_true", |
| default=False, |
| help="echo executed commands", |
| ) |
| parser.add_argument( |
| "--vv", |
| action="store_true", |
| default=False, |
| help="run cargo with -vv instead of default -v", |
| ) |
| parser.add_argument( |
| "--dump-config-and-exit", |
| type=str, |
| help=( |
| "Dump command-line arguments (minus this flag) to a config file and" |
| " exit. This is intended to help migrate from command line options " |
| "to config files." |
| ), |
| ) |
| parser.add_argument( |
| "--config", |
| type=str, |
| help=( |
| "Load command-line options from the given config file. Options in " |
| "this file will override those passed on the command line." |
| ), |
| ) |
| return parser |
| |
| |
| def parse_args(parser): |
| """Parses command-line options.""" |
| args = parser.parse_args() |
| # Use the values specified in a config file if one was found. |
| if args.config: |
| with open(args.config, "r", encoding="utf-8") as f: |
| config = json.load(f) |
| args_dict = vars(args) |
| for arg in config: |
| args_dict[arg.replace("-", "_")] = config[arg] |
| return args |
| |
| |
| def dump_config(parser, args): |
| """Writes the non-default command-line options to the specified file.""" |
| args_dict = vars(args) |
| # Filter out the arguments that have their default value. |
| # Also filter certain "temporary" arguments. |
| non_default_args = {} |
| for arg in args_dict: |
| if ( |
| args_dict[arg] != parser.get_default(arg) |
| and arg != "dump_config_and_exit" |
| and arg != "config" |
| and arg != "cargo_bin" |
| ): |
| non_default_args[arg.replace("_", "-")] = args_dict[arg] |
| # Write to the specified file. |
| with open(args.dump_config_and_exit, "w", encoding="utf-8") as f: |
| json.dump(non_default_args, f, indent=2, sort_keys=True) |
| |
| |
| def main(): |
| parser = get_parser() |
| args = parse_args(parser) |
| if not args.run: # default is dry-run |
| print(DRY_RUN_NOTE) |
| if args.dump_config_and_exit: |
| dump_config(parser, args) |
| else: |
| Runner(args).run_cargo().gen_rules().apply_patch() |
| |
| |
| if __name__ == "__main__": |
| main() |
