benchmarks/linker_relocation/regen/dump_relocs.py - platform/bionic.git - Git at Google

 #!/usr/bin/env python3
 #
 # Copyright (C) 2019 The Android Open Source Project
 # All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions
 # are met:
 #  * Redistributions of source code must retain the above copyright
 #    notice, this list of conditions and the following disclaimer.
 #  * Redistributions in binary form must reproduce the above copyright
 #    notice, this list of conditions and the following disclaimer in
 #    the documentation and/or other materials provided with the
 #    distribution.
 #
 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
 # FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
 # COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
 # INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
 # BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
 # OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
 # AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
 # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
 # OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 # SUCH DAMAGE.

 # Scan an ELF file and its tree of DT_NEEDED ELF files, and dump out a JSON file listing:
 #  - each ELF file
 #  - its DT_NEEDED entries
 #  - its defined symbols
 #  - its relocations

 import argparse
 import json
 import os
 import re
 import shlex
 import shutil
 import subprocess
 import sys
 import tempfile
 import textwrap
 import typing
 from enum import Enum
 from typing import Any, Set, List, Dict, Optional
 from subprocess import PIPE, DEVNULL
 from pathlib import Path

 from common_types import LoadedLibrary, SymBind, SymKind, DynSymbol, DynSymbols, Relocations, \
     SymbolRef, bfs_walk, elf_tree_to_json


 g_readelf_cache: Dict[str, str] = {}
 g_path_to_soname_cache: Dict[Path, str] = {}

 def do_readelf_query(arguments: List[str]) -> List[str]:
     cmdline = ['llvm-readelf'] + arguments
     key = repr(cmdline)
     if key in g_readelf_cache: return g_readelf_cache[key].splitlines()
     out = subprocess.run(cmdline, check=True, stdout=PIPE).stdout.decode()
     g_readelf_cache[key] = out
     return out.splitlines()


 def get_elf_soname(path: Path) -> str:
     if path in g_path_to_soname_cache: return g_path_to_soname_cache[path]
     out = do_readelf_query(['-d', str(path)])
     for line in out:
         m = re.search(r'\(SONAME\)\s+Library soname: \[(.+)\]$', line)
         if not m: continue
         result = m.group(1)
         break
     else:
         result = os.path.basename(path)
     g_path_to_soname_cache[path] = result
     return result


 def get_elf_needed(path: Path) -> List[str]:
     result = []
     out = do_readelf_query(['-d', str(path)])
     for line in out:
         m = re.search(r'\(NEEDED\)\s+Shared library: \[(.+)\]$', line)
         if not m: continue
         result.append(m.group(1))
     return result


 kSymbolMatcher = re.compile(r'''
     \s+ (\d+) : \s*                 # number
     [0-9a-f]+ \s+                   # value
     [0-9a-f]+ \s+                   # size
     (FUNC|IFUNC|OBJECT|NOTYPE) \s+  # type
     (GLOBAL|WEAK) \s+               # bind
     \w+ \s+                         # vis
     (\d+|UND) \s+                   # ndx
     ([\.\w]+)                       # name
     (?:(@@?)(\w+))?                 # version
     $
 ''', re.VERBOSE)


 def get_dyn_symbols(path: Path) -> DynSymbols:
     kind_lookup = {
         'FUNC': SymKind.Func,
         'IFUNC': SymKind.Func,
         'OBJECT': SymKind.Var,
         'NOTYPE': SymKind.Func,
     }
     bind_lookup = { 'GLOBAL': SymBind.Global, 'WEAK': SymBind.Weak }

     result = {}
     out = do_readelf_query(['--dyn-syms', str(path)])
     for line in out:
         m = kSymbolMatcher.match(line)
         if not m:
             # gLinux currently has a version of llvm-readelf whose output is very different from
             # the current versions of llvm-readelf (or GNU readelf).
             if 'Symbol table of .gnu.hash for image:' in line:
                 sys.exit(f'error: obsolete version of llvm-readelf')
             continue

         num, kind, bind, ndx, name, ver_type, ver_name = m.groups()

         if name == '__cfi_check':
             # The linker gives an error like:
             #    CANNOT LINK EXECUTABLE "/data/local/tmp/out-linker-bench/b_libandroid_servers": unaligned __cfi_check in the library "(null)"
             # I am probably breaking some kind of CFI invariant, so strip these out for now.
             continue

         result[int(num)] = DynSymbol(name, kind_lookup[kind], bind_lookup[bind], ndx != 'UND',
                                      ver_type, ver_name)

     return result


 kRelocationMatcher = re.compile(r'''
     ([0-9a-f]+) \s+     # offset
     ([0-9a-f]+) \s+     # info
     (\w+)               # type
     (?:
         \s+ [0-9a-f]+ \s+       # symbol value
         ([\.\w]+)               # symbol name
         (?: @@? ([\.\w]+) )?    # version
     )?
     \b
 ''', re.VERBOSE)


 def scan_relocations(path: Path, syms: DynSymbols) -> Relocations:
     result: Relocations = Relocations()
     out = do_readelf_query(['-r', str(path)])
     for line in out:
         m = kRelocationMatcher.match(line)
         if not m: continue

         offset_str, info_str, reloc_name, sym_name, ver = m.groups()

         if len(offset_str) == 8:
             offset = int(offset_str, 16) // 4
             sym_idx = int(info_str, 16) >> 8
         elif len(offset_str) == 16:
             offset = int(offset_str, 16) // 8
             sym_idx = int(info_str, 16) >> 32
         else:
             sys.exit(f'error: invalid offset length: {repr(offset_str)}')

         # TODO: R_ARM_IRELATIVE doesn't work, so skip it.
         if reloc_name == 'R_ARM_IRELATIVE': continue

         if reloc_name in ['R_ARM_RELATIVE', 'R_AARCH64_RELATIVE']:
             assert sym_name is None
             result.relative.append(offset)
         else:
             if sym_name is None:
                 sys.exit(f'error: missing symbol for reloc {m.groups()} in {path}')

             is_weak = syms[sym_idx].bind == SymBind.Weak
             symbol = SymbolRef(sym_name, is_weak, ver)

             if reloc_name in ['R_ARM_JUMP_SLOT', 'R_AARCH64_JUMP_SLOT']:
                 result.jump_slots.append(symbol)
             elif reloc_name in ['R_ARM_GLOB_DAT', 'R_AARCH64_GLOB_DAT']:
                 result.got.append(symbol)
             elif reloc_name in ['R_ARM_ABS32', 'R_AARCH64_ABS64']:
                 result.symbolic.append((offset, symbol))
             else:
                 sys.exit(f'error: unrecognized reloc {m.groups()} in {path}')

     return result


 def load_elf_tree(search_path: List[Path], path: Path) -> LoadedLibrary:

     libraries: Dict[str, LoadedLibrary] = {}

     def find_library(needed: str) -> Optional[LoadedLibrary]:
         nonlocal libraries

         if needed in libraries: return libraries[needed]

         for candidate_dir in search_path:
             candidate_path = candidate_dir / needed
             if candidate_path.exists():
                 return load(candidate_path)

         sys.exit(f'error: missing DT_NEEDED lib {needed}!')

     def load(path: Path) -> LoadedLibrary:
         nonlocal libraries

         lib = LoadedLibrary()
         lib.soname = get_elf_soname(path)
         if lib.soname in libraries: sys.exit(f'soname already loaded: {lib.soname}')
         libraries[lib.soname] = lib

         lib.syms = get_dyn_symbols(path)
         lib.rels = scan_relocations(path, lib.syms)

         for needed in get_elf_needed(path):
             needed_lib = find_library(needed)
             if needed_lib is not None:
                 lib.needed.append(needed_lib)

         return lib

     return load(path)


 def main() -> None:
     parser = argparse.ArgumentParser()
     parser.add_argument('input', type=str)
     parser.add_argument('output', type=str)
     parser.add_argument('-L', dest='search_path', metavar='PATH', action='append', type=str, default=[])

     args = parser.parse_args()
     search_path = [Path(p) for p in args.search_path]

     with open(Path(args.output), 'w') as f:
         root = load_elf_tree(search_path, Path(args.input))
         json.dump(elf_tree_to_json(root), f, sort_keys=True, indent=2)


 if __name__ == '__main__':
     main()
	#!/usr/bin/env python3
	#
	# Copyright (C) 2019 The Android Open Source Project
	# All rights reserved.
	#
	# Redistribution and use in source and binary forms, with or without
	# modification, are permitted provided that the following conditions
	# are met:
	# * Redistributions of source code must retain the above copyright
	# notice, this list of conditions and the following disclaimer.
	# * Redistributions in binary form must reproduce the above copyright
	# notice, this list of conditions and the following disclaimer in
	# the documentation and/or other materials provided with the
	# distribution.
	#
	# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
	# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
	# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
	# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
	# COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
	# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
	# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
	# OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
	# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
	# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
	# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
	# SUCH DAMAGE.

	# Scan an ELF file and its tree of DT_NEEDED ELF files, and dump out a JSON file listing:
	# - each ELF file
	# - its DT_NEEDED entries
	# - its defined symbols
	# - its relocations

	import argparse
	import json
	import os
	import re
	import shlex
	import shutil
	import subprocess
	import sys
	import tempfile
	import textwrap
	import typing
	from enum import Enum
	from typing import Any, Set, List, Dict, Optional
	from subprocess import PIPE, DEVNULL
	from pathlib import Path

	from common_types import LoadedLibrary, SymBind, SymKind, DynSymbol, DynSymbols, Relocations, \
	SymbolRef, bfs_walk, elf_tree_to_json


	g_readelf_cache: Dict[str, str] = {}
	g_path_to_soname_cache: Dict[Path, str] = {}

	def do_readelf_query(arguments: List[str]) -> List[str]:
	cmdline = ['llvm-readelf'] + arguments
	key = repr(cmdline)
	if key in g_readelf_cache: return g_readelf_cache[key].splitlines()
	out = subprocess.run(cmdline, check=True, stdout=PIPE).stdout.decode()
	g_readelf_cache[key] = out
	return out.splitlines()


	def get_elf_soname(path: Path) -> str:
	if path in g_path_to_soname_cache: return g_path_to_soname_cache[path]
	out = do_readelf_query(['-d', str(path)])
	for line in out:
	m = re.search(r'\(SONAME\)\s+Library soname: \[(.+)\]$', line)
	if not m: continue
	result = m.group(1)
	break
	else:
	result = os.path.basename(path)
	g_path_to_soname_cache[path] = result
	return result


	def get_elf_needed(path: Path) -> List[str]:
	result = []
	out = do_readelf_query(['-d', str(path)])
	for line in out:
	m = re.search(r'\(NEEDED\)\s+Shared library: \[(.+)\]$', line)
	if not m: continue
	result.append(m.group(1))
	return result


	kSymbolMatcher = re.compile(r'''
	\s+ (\d+) : \s* # number
	[0-9a-f]+ \s+ # value
	[0-9a-f]+ \s+ # size
	(FUNC\|IFUNC\|OBJECT\|NOTYPE) \s+ # type
	(GLOBAL\|WEAK) \s+ # bind
	\w+ \s+ # vis
	(\d+\|UND) \s+ # ndx
	([\.\w]+) # name
	(?:(@@?)(\w+))? # version
	$
	''', re.VERBOSE)


	def get_dyn_symbols(path: Path) -> DynSymbols:
	kind_lookup = {
	'FUNC': SymKind.Func,
	'IFUNC': SymKind.Func,
	'OBJECT': SymKind.Var,
	'NOTYPE': SymKind.Func,
	}
	bind_lookup = { 'GLOBAL': SymBind.Global, 'WEAK': SymBind.Weak }

	result = {}
	out = do_readelf_query(['--dyn-syms', str(path)])
	for line in out:
	m = kSymbolMatcher.match(line)
	if not m:
	# gLinux currently has a version of llvm-readelf whose output is very different from
	# the current versions of llvm-readelf (or GNU readelf).
	if 'Symbol table of .gnu.hash for image:' in line:
	sys.exit(f'error: obsolete version of llvm-readelf')
	continue

	num, kind, bind, ndx, name, ver_type, ver_name = m.groups()

	if name == '__cfi_check':
	# The linker gives an error like:
	# CANNOT LINK EXECUTABLE "/data/local/tmp/out-linker-bench/b_libandroid_servers": unaligned __cfi_check in the library "(null)"
	# I am probably breaking some kind of CFI invariant, so strip these out for now.
	continue

	result[int(num)] = DynSymbol(name, kind_lookup[kind], bind_lookup[bind], ndx != 'UND',
	ver_type, ver_name)

	return result


	kRelocationMatcher = re.compile(r'''
	([0-9a-f]+) \s+ # offset
	([0-9a-f]+) \s+ # info
	(\w+) # type
	(?:
	\s+ [0-9a-f]+ \s+ # symbol value
	([\.\w]+) # symbol name
	(?: @@? ([\.\w]+) )? # version
	)?
	\b
	''', re.VERBOSE)


	def scan_relocations(path: Path, syms: DynSymbols) -> Relocations:
	result: Relocations = Relocations()
	out = do_readelf_query(['-r', str(path)])
	for line in out:
	m = kRelocationMatcher.match(line)
	if not m: continue

	offset_str, info_str, reloc_name, sym_name, ver = m.groups()

	if len(offset_str) == 8:
	offset = int(offset_str, 16) // 4
	sym_idx = int(info_str, 16) >> 8
	elif len(offset_str) == 16:
	offset = int(offset_str, 16) // 8
	sym_idx = int(info_str, 16) >> 32
	else:
	sys.exit(f'error: invalid offset length: {repr(offset_str)}')

	# TODO: R_ARM_IRELATIVE doesn't work, so skip it.
	if reloc_name == 'R_ARM_IRELATIVE': continue

	if reloc_name in ['R_ARM_RELATIVE', 'R_AARCH64_RELATIVE']:
	assert sym_name is None
	result.relative.append(offset)
	else:
	if sym_name is None:
	sys.exit(f'error: missing symbol for reloc {m.groups()} in {path}')

	is_weak = syms[sym_idx].bind == SymBind.Weak
	symbol = SymbolRef(sym_name, is_weak, ver)

	if reloc_name in ['R_ARM_JUMP_SLOT', 'R_AARCH64_JUMP_SLOT']:
	result.jump_slots.append(symbol)
	elif reloc_name in ['R_ARM_GLOB_DAT', 'R_AARCH64_GLOB_DAT']:
	result.got.append(symbol)
	elif reloc_name in ['R_ARM_ABS32', 'R_AARCH64_ABS64']:
	result.symbolic.append((offset, symbol))
	else:
	sys.exit(f'error: unrecognized reloc {m.groups()} in {path}')

	return result


	def load_elf_tree(search_path: List[Path], path: Path) -> LoadedLibrary:

	libraries: Dict[str, LoadedLibrary] = {}

	def find_library(needed: str) -> Optional[LoadedLibrary]:
	nonlocal libraries

	if needed in libraries: return libraries[needed]

	for candidate_dir in search_path:
	candidate_path = candidate_dir / needed
	if candidate_path.exists():
	return load(candidate_path)

	sys.exit(f'error: missing DT_NEEDED lib {needed}!')

	def load(path: Path) -> LoadedLibrary:
	nonlocal libraries

	lib = LoadedLibrary()
	lib.soname = get_elf_soname(path)
	if lib.soname in libraries: sys.exit(f'soname already loaded: {lib.soname}')
	libraries[lib.soname] = lib

	lib.syms = get_dyn_symbols(path)
	lib.rels = scan_relocations(path, lib.syms)

	for needed in get_elf_needed(path):
	needed_lib = find_library(needed)
	if needed_lib is not None:
	lib.needed.append(needed_lib)

	return lib

	return load(path)


	def main() -> None:
	parser = argparse.ArgumentParser()
	parser.add_argument('input', type=str)
	parser.add_argument('output', type=str)
	parser.add_argument('-L', dest='search_path', metavar='PATH', action='append', type=str, default=[])

	args = parser.parse_args()
	search_path = [Path(p) for p in args.search_path]

	with open(Path(args.output), 'w') as f:
	root = load_elf_tree(search_path, Path(args.input))
	json.dump(elf_tree_to_json(root), f, sort_keys=True, indent=2)


	if __name__ == '__main__':
	main()