blob: 31cecc18d44dbf05627263d68fc4a8c1b0d78ed4 [file] [log] [blame]
#!/usr/bin/env python3
#
# Copyright (C) 2019 The Android Open Source Project
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
import argparse
import collections
import functools
import itertools
import os
import re
import subprocess
import sys
_ALWAYS_INCLUDED = [
"module_layout", # is exported even if CONFIG_TRIM_UNUSED_KSYMS is enabled
"__put_task_struct", # this allows us to keep `struct task_struct` stable
]
_ABIGAIL_HEADER = "[abi_symbol_list]"
def symbol_sort(symbols):
# use the method that `sort` uses: case insensitive and ignoring
# underscores, that keeps symbols with related name close to each other.
# yeah, that is a bit brute force, but it gets the job done
def __key(a):
"""Creates a key for comparison of symbols."""
# We want to sort underscore prefixed symbols along with those without, but
# before them. Hence add a trailing underscore for every missing leading
# one and strip all others.
# E.g. __blk_mq_end_request, _blk_mq_end_request, blk_mq_end_request get
# replaced by blkmqendrequest, blkmqendrequest_, blkmqendrequest__ and
# compared lexicographically.
# if the caller passes None or an empty string something is odd, so assert
# and ignore if asserts are disabled as we do not need to deal with that
assert (a)
if not a:
return a
tmp = a.lower()
for idx, c in enumerate(tmp):
if c != "_":
break
return (tmp.replace("_", "") + (5 - idx) * "_", a)
return sorted(set(symbols), key=__key)
def find_binaries(directory):
"""Locates vmlinux and kernel modules (*.ko)."""
vmlinux = None
modules = []
for root, dirs, files in os.walk(directory):
for file in files:
if file.endswith(".ko"):
modules.append(os.path.join(root, file))
elif file == "vmlinux":
vmlinux = os.path.join(root, file)
return vmlinux, modules
def extract_undefined_symbols(modules):
"""Extracts undefined symbols from a list of module files."""
# yes, we could pass all of them to nm, but I want to avoid hitting shell
# limits with long lists of modules
result = {}
for module in sorted(modules):
symbols = []
out = subprocess.check_output(["llvm-nm", "--undefined-only", module],
stderr=subprocess.DEVNULL).decode("ascii")
for line in out.splitlines():
symbols.append(line.strip().split()[1])
result[os.path.basename(module)] = symbol_sort(symbols)
return result
def extract_exported_symbols(binary):
"""Extracts the ksymtab exported symbols from a kernel binary."""
symbols = []
out = subprocess.check_output(["llvm-nm", "--defined-only", binary],
stderr=subprocess.DEVNULL).decode("ascii")
for line in out.splitlines():
pos = line.find(" __ksymtab_")
if pos != -1:
symbols.append(line[pos + len(" __ksymtab_"):])
return symbol_sort(symbols)
def extract_generic_exports(vmlinux, modules):
"""Extracts the ksymtab exported symbols from vmlinux and a set of modules."""
symbols = extract_exported_symbols(vmlinux)
for module in modules:
symbols.extend(extract_exported_symbols(module))
return symbols
def extract_exported_in_modules(modules):
"""Extracts the ksymtab exported symbols for a list of kernel modules."""
return {module: extract_exported_symbols(module) for module in modules}
def report_missing(module_symbols, exported):
"""Reports missing symbols that are undefined, but not known in any binary."""
for module, symbols in module_symbols.items():
for symbol in symbols:
if symbol not in exported:
print("Symbol {} required by {} but not provided".format(
symbol, module))
def read_symbol_list(symbol_list):
"""Reads a previously created libabigail symbol symbol list."""
symbols = []
with open(symbol_list) as wl:
for line in [l.strip() for l in wl]:
if not line or line.startswith("#") or line.startswith("["):
continue
symbols.append(line)
return symbols
def create_symbol_list(symbol_list, undefined_symbols, exported,
emit_module_symbol_lists, module_grouping,
additions_only):
"""Creates a symbol symbol list for libabigail."""
precious_symbols = set()
if additions_only:
precious_symbols.update(read_symbol_list(symbol_list))
symbol_counter = collections.Counter(
itertools.chain.from_iterable(undefined_symbols.values()))
with open(symbol_list, "w") as wl:
common_wl_section = symbol_sort([
symbol for symbol, count in symbol_counter.items()
if (count > 1 or not module_grouping) and symbol in exported
] + _ALWAYS_INCLUDED)
# write the header
wl.write(_ABIGAIL_HEADER)
wl.write("\n")
if module_grouping:
wl.write("# commonly used symbols\n")
wl.write(" ")
wl.write("\n ".join(common_wl_section))
wl.write("\n")
precious_symbols.difference_update(common_wl_section)
for module, symbols in undefined_symbols.items():
if emit_module_symbol_lists:
mod_wl_file = symbol_list + "_" + os.path.splitext(module)[0]
with open(mod_wl_file, "w") as mod_wl:
# write the header
mod_wl.write(_ABIGAIL_HEADER)
mod_wl.write("\n ")
mod_wl.write("\n ".join([s for s in symbols if s in exported]))
mod_wl.write("\n")
new_wl_section = symbol_sort([
symbol for symbol in symbols
if symbol in exported and symbol not in common_wl_section
])
if not new_wl_section:
continue
wl.write("\n# required by {}\n ".format(module))
wl.write("\n ".join(new_wl_section))
wl.write("\n")
precious_symbols.difference_update(new_wl_section)
if precious_symbols:
wl.write("\n# preserved by --additions-only\n ")
wl.write("\n ".join(symbol_sort(precious_symbols)))
wl.write("\n")
def main():
"""Extracts the required symbols for a directory full of kernel modules."""
parser = argparse.ArgumentParser()
parser.add_argument(
"directory",
nargs="?",
default=os.getcwd(),
help="the directory to search for kernel binaries")
parser.add_argument(
"--skip-report-missing",
action="store_false",
dest="report_missing",
help="Do not report symbols required by modules, but missing from vmlinux"
)
parser.add_argument(
"--include-module-exports",
action="store_true",
help="Include inter-module symbols")
parser.add_argument(
"--full-gki-abi",
action="store_true",
help="Assume all vmlinux and GKI module symbols are part of the ABI")
parser.add_argument(
"--symbol-list", "--whitelist",
help="The symbol list to create")
parser.add_argument(
"--additions-only",
action="store_true",
help="Read the existing symbol list and ensure no symbols get removed")
parser.add_argument(
"--print-modules",
action="store_true",
help="Emit the names of the processed modules")
parser.add_argument(
"--emit-module-symbol-lists", "--emit-module-whitelists",
action="store_true",
help="Emit a separate symbol list for each module")
parser.add_argument(
"--skip-module-grouping",
action="store_false",
dest="module_grouping",
help="Do not group symbols by module.")
parser.add_argument(
"--module-filter",
action="append",
dest="module_filters",
help="Only process modules matching the filter. Can be passed multiple times."
)
parser.add_argument(
"--gki-modules",
help="List of GKI modules which must be provided when the search directory contains both vendor and GKI modules")
args = parser.parse_args()
if not os.path.isdir(args.directory):
print("Expected a directory to search for binaries, but got %s" %
args.directory)
return 1
if args.emit_module_symbol_lists and not args.symbol_list:
print("Emitting module symbol lists requires the --symbol-list parameter.")
return 1
if args.symbol_list is None:
args.symbol_list = "/dev/stdout"
# Locate the Kernel Binaries
vmlinux, modules = find_binaries(args.directory)
if args.module_filters:
modules = [
mod for mod in modules if any(
[re.search(f, os.path.basename(mod)) for f in args.module_filters])
]
# Partition vendor and GKI modules in two lists
gki_modules = []
if args.gki_modules is not None:
with open(args.gki_modules) as f:
gki_modules = [ os.path.basename(mod) for mod in f.read().splitlines() ]
gki_modules = [ mod for mod in modules if os.path.basename(mod) in gki_modules ]
modules = [ mod for mod in modules if mod not in gki_modules ]
if vmlinux is None or not os.path.isfile(vmlinux):
print("Could not find a suitable vmlinux file.")
return 1
# Get required symbols of all modules
undefined_symbols = extract_undefined_symbols(modules)
# Get the actually defined and exported symbols
generic_exports = extract_generic_exports(vmlinux, gki_modules)
local_exports = extract_exported_in_modules(modules)
# Build the list of all exported symbols (generic and local)
all_exported = list(
itertools.chain.from_iterable(local_exports.values()))
all_exported.extend(generic_exports)
all_exported = set(all_exported)
# For sanity, check for inconsistencies between required and exported symbols
# Do not do this analysis if module_filters are in place as likely
# inter-module dependencies are broken by this.
if args.report_missing and not args.module_filters:
report_missing(undefined_symbols, all_exported)
# If specified, create the symbol list
if args.symbol_list:
create_symbol_list(
args.symbol_list,
{ "full-gki-abi": generic_exports } if args.full_gki_abi else undefined_symbols,
all_exported if args.include_module_exports else generic_exports,
args.emit_module_symbol_lists,
args.module_grouping,
args.additions_only)
if args.print_modules:
print("These modules have been considered when creating the symbol list:")
print(" " +
"\n ".join(sorted([os.path.basename(mod) for mod in modules])))
if __name__ == "__main__":
sys.exit(main())