| # Copyright 2013 The Chromium Authors. All rights reserved. |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| |
| """Takes the same arguments as Windows link.exe, and a definition of libraries |
| to split into subcomponents. Does multiple passes of link.exe invocation to |
| determine exports between parts and generates .def and import libraries to |
| cause symbols to be available to other parts.""" |
| |
| import _winreg |
| import ctypes |
| import os |
| import re |
| import shutil |
| import subprocess |
| import sys |
| import tempfile |
| |
| |
| BASE_DIR = os.path.dirname(os.path.abspath(__file__)) |
| |
| |
| # This can be set to ignore data exports. The resulting DLLs will probably not |
| # run, but at least they can be generated. The log of data exports will still |
| # be output. |
| IGNORE_DATA = 0 |
| |
| |
| def Log(message): |
| print 'split_link:', message |
| |
| |
| def GetFlagsAndInputs(argv): |
| """Parses the command line intended for link.exe and return the flags and |
| input files.""" |
| rsp_expanded = [] |
| for arg in argv: |
| if arg[0] == '@': |
| with open(arg[1:]) as rsp: |
| rsp_expanded.extend(rsp.read().splitlines()) |
| else: |
| rsp_expanded.append(arg) |
| |
| # Use CommandLineToArgvW so we match link.exe parsing. |
| try: |
| size = ctypes.c_int() |
| ptr = ctypes.windll.shell32.CommandLineToArgvW( |
| ctypes.create_unicode_buffer(' '.join(rsp_expanded)), |
| ctypes.byref(size)) |
| ref = ctypes.c_wchar_p * size.value |
| raw = ref.from_address(ptr) |
| args = [arg for arg in raw] |
| finally: |
| ctypes.windll.kernel32.LocalFree(ptr) |
| |
| inputs = [] |
| flags = [] |
| intermediate_manifest = '' |
| for arg in args: |
| lower_arg = arg.lower() |
| # We'll be replacing these ourselves. |
| if lower_arg.startswith('/out:'): |
| continue |
| if lower_arg.startswith('/manifestfile:'): |
| intermediate_manifest = arg[arg.index(':')+1:] |
| continue |
| if lower_arg.startswith('/pdb:'): |
| continue |
| if (not lower_arg.startswith('/') and |
| lower_arg.endswith(('.obj', '.lib', '.res'))): |
| inputs.append(arg) |
| else: |
| flags.append(arg) |
| |
| return flags, inputs, intermediate_manifest |
| |
| |
| def GetRegistryValue(subkey): |
| try: |
| val = _winreg.QueryValue(_winreg.HKEY_CURRENT_USER, |
| 'Software\\Chromium\\' + subkey) |
| if os.path.exists(val): |
| return val |
| except WindowsError: |
| pass |
| |
| raise SystemExit("Couldn't read from registry") |
| |
| |
| def GetOriginalLinkerPath(): |
| return GetRegistryValue('split_link_installed') |
| |
| |
| def GetMtPath(): |
| return GetRegistryValue('split_link_mt_path') |
| |
| |
| def PartFor(input_file, description_parts, description_all): |
| """Determines which part a given link input should be put into (or all).""" |
| # Check if it should go in all parts. |
| input_file = input_file.lower() |
| if any(re.search(spec, input_file) for spec in description_all): |
| return -1 |
| # Or pick which particular one it belongs in. |
| for i, spec_list in enumerate(description_parts): |
| if any(re.search(spec, input_file) for spec in spec_list): |
| return i |
| raise ValueError("couldn't find location for %s" % input_file) |
| |
| |
| def ParseOutExternals(output): |
| """Given the stdout of link.exe, parses the error messages to retrieve all |
| symbols that are unresolved.""" |
| result = set() |
| # Styles of messages for unresolved externals, and a boolean to indicate |
| # whether the error message emits the symbols with or without a leading |
| # underscore. |
| unresolved_regexes = [ |
| (re.compile(r' : error LNK2019: unresolved external symbol ".*" \((.*)\)' |
| r' referenced in function'), |
| False), |
| (re.compile(r' : error LNK2001: unresolved external symbol ".*" \((.*)\)$'), |
| False), |
| (re.compile(r' : error LNK2019: unresolved external symbol (.*)' |
| r' referenced in function '), |
| True), |
| (re.compile(r' : error LNK2001: unresolved external symbol (.*)$'), |
| True), |
| ] |
| for line in output.splitlines(): |
| line = line.strip() |
| for regex, strip_leading_underscore in unresolved_regexes: |
| mo = regex.search(line) |
| if mo: |
| if strip_leading_underscore: |
| result.add(mo.group(1)[1:]) |
| else: |
| result.add(mo.group(1)) |
| break |
| |
| mo = re.search(r'fatal error LNK1120: (\d+) unresolved externals', output) |
| # Make sure we have the same number that the linker thinks we have. |
| if mo is None and result: |
| raise SystemExit(output) |
| if len(result) != int(mo.group(1)): |
| print output |
| print 'Expecting %d, got %d' % (int(mo.group(1)), len(result)) |
| assert len(result) == int(mo.group(1)) |
| return sorted(result) |
| |
| |
| def AsCommandLineArgs(items): |
| """Intended for output to a response file. Quotes all arguments.""" |
| return '\n'.join('"' + x + '"' for x in items) |
| |
| |
| def OutputNameForIndex(index): |
| """Gets the final output DLL name, given a zero-based index.""" |
| if index == 0: |
| return "chrome.dll" |
| else: |
| return 'chrome%d.dll' % index |
| |
| |
| def ManifestNameForIndex(index): |
| return OutputNameForIndex(index) + '.intermediate.manifest' |
| |
| |
| def PdbNameForIndex(index): |
| return OutputNameForIndex(index) + '.pdb' |
| |
| |
| def RunLinker(flags, index, inputs, phase, intermediate_manifest): |
| """Invokes the linker and returns the stdout, returncode and target name.""" |
| rspfile = 'part%d_%s.rsp' % (index, phase) |
| with open(rspfile, 'w') as f: |
| print >> f, AsCommandLineArgs(inputs) |
| print >> f, AsCommandLineArgs(flags) |
| output_name = OutputNameForIndex(index) |
| manifest_name = ManifestNameForIndex(index) |
| print >> f, '/ENTRY:ChromeEmptyEntry@12' |
| print >> f, '/OUT:' + output_name |
| print >> f, '/MANIFESTFILE:' + manifest_name |
| print >> f, '/PDB:' + PdbNameForIndex(index) |
| # Log('[[[\n' + open(rspfile).read() + '\n]]]') |
| link_exe = GetOriginalLinkerPath() |
| popen = subprocess.Popen([link_exe, '@' + rspfile], stdout=subprocess.PIPE) |
| stdout, _ = popen.communicate() |
| if index == 0 and popen.returncode == 0 and intermediate_manifest: |
| # Hack for ninja build. After the linker runs, it does some manifest |
| # things and expects there to be a file in this location. We just put it |
| # there so it's happy. This is a no-op. |
| if os.path.isdir(os.path.dirname(intermediate_manifest)): |
| shutil.copyfile(manifest_name, intermediate_manifest) |
| return stdout, popen.returncode, output_name |
| |
| |
| def GetLibObjList(lib): |
| """Gets the list of object files contained in a .lib.""" |
| link_exe = GetOriginalLinkerPath() |
| popen = subprocess.Popen( |
| [link_exe, '/lib', '/nologo', '/list', lib], stdout=subprocess.PIPE) |
| stdout, _ = popen.communicate() |
| return stdout.splitlines() |
| |
| |
| def ExtractObjFromLib(lib, obj): |
| """Extracts a .obj file contained in a .lib file. Returns the absolute path |
| a temp file.""" |
| link_exe = GetOriginalLinkerPath() |
| temp = tempfile.NamedTemporaryFile( |
| prefix='split_link_', suffix='.obj', delete=False) |
| temp.close() |
| subprocess.check_call([ |
| link_exe, '/lib', '/nologo', '/extract:' + obj, lib, '/out:' + temp.name]) |
| return temp.name |
| |
| |
| def Unmangle(export): |
| "Returns the human-presentable name of a mangled symbol.""" |
| # Use dbghelp.dll to demangle the name. |
| # TODO(scottmg): Perhaps a simple cache? Seems pretty fast though. |
| UnDecorateSymbolName = ctypes.windll.dbghelp.UnDecorateSymbolName |
| buffer_size = 2048 |
| output_string = ctypes.create_string_buffer(buffer_size) |
| if not UnDecorateSymbolName( |
| export, ctypes.byref(output_string), buffer_size, 0): |
| raise ctypes.WinError() |
| return output_string.value |
| |
| |
| def IsDataDefinition(export): |
| """Determines if a given name is data rather than a function. Always returns |
| False for C-style (as opposed to C++-style names).""" |
| if export[0] != '?': |
| return False |
| |
| # If it contains a '(' we assume it's a function. |
| return '(' not in Unmangle(export) |
| |
| |
| def GenerateDefFiles(unresolved_by_part): |
| """Given a list of unresolved externals, generates a .def file that will |
| cause all those symbols to be exported.""" |
| deffiles = [] |
| Log('generating .def files') |
| for i, part in enumerate(unresolved_by_part): |
| deffile = 'part%d.def' % i |
| with open(deffile, 'w') as f: |
| print >> f, 'LIBRARY %s' % OutputNameForIndex(i) |
| print >> f, 'EXPORTS' |
| for j, part in enumerate(unresolved_by_part): |
| if i == j: |
| continue |
| is_data = \ |
| [' DATA' if IsDataDefinition(export) and not IGNORE_DATA else '' |
| for export in part] |
| print >> f, '\n'.join(' ' + export + data |
| for export, data in zip(part, is_data)) |
| deffiles.append(deffile) |
| return deffiles |
| |
| |
| def BuildImportLibs(flags, inputs_by_part, deffiles): |
| """Runs the linker to generate an import library.""" |
| import_libs = [] |
| Log('building import libs') |
| for i, (inputs, deffile) in enumerate(zip(inputs_by_part, deffiles)): |
| libfile = 'part%d.lib' % i |
| flags_with_implib_and_deffile = flags + ['/IMPLIB:%s' % libfile, |
| '/DEF:%s' % deffile] |
| RunLinker(flags_with_implib_and_deffile, i, inputs, 'implib', None) |
| import_libs.append(libfile) |
| return import_libs |
| |
| |
| def AttemptLink(flags, inputs_by_part, unresolved_by_part, deffiles, |
| import_libs, intermediate_manifest): |
| """Tries to run the linker for all parts using the current round of |
| generated import libs and .def files. If the link fails, updates the |
| unresolved externals list per part.""" |
| dlls = [] |
| all_succeeded = True |
| new_externals = [] |
| Log('unresolveds now: %r' % [len(part) for part in unresolved_by_part]) |
| for i, (inputs, deffile) in enumerate(zip(inputs_by_part, deffiles)): |
| Log('running link, part %d' % i) |
| others_implibs = import_libs[:] |
| others_implibs.pop(i) |
| inputs_with_implib = inputs + filter(lambda x: x, others_implibs) |
| if deffile: |
| flags = flags + ['/DEF:%s' % deffile, '/LTCG'] |
| stdout, rc, output = RunLinker( |
| flags, i, inputs_with_implib, 'final', intermediate_manifest) |
| if rc != 0: |
| all_succeeded = False |
| new_externals.append(ParseOutExternals(stdout)) |
| else: |
| new_externals.append([]) |
| dlls.append(output) |
| combined_externals = [sorted(set(prev) | set(new)) |
| for prev, new in zip(unresolved_by_part, new_externals)] |
| return all_succeeded, dlls, combined_externals |
| |
| |
| def ExtractSubObjsTargetedAtAll( |
| inputs, |
| num_parts, |
| description_parts, |
| description_all, |
| description_all_from_libs): |
| """For (lib, obj) tuples in the all_from_libs section, extract the obj out of |
| the lib and added it to inputs. Returns a list of lists for which part the |
| extracted obj belongs in (which is whichever the .lib isn't in).""" |
| by_parts = [[] for _ in range(num_parts)] |
| for lib_spec, obj_spec in description_all_from_libs: |
| for input_file in inputs: |
| if re.search(lib_spec, input_file): |
| objs = GetLibObjList(input_file) |
| match_count = 0 |
| for obj in objs: |
| if re.search(obj_spec, obj, re.I): |
| extracted_obj = ExtractObjFromLib(input_file, obj) |
| #Log('extracted %s (%s %s)' % (extracted_obj, input_file, obj)) |
| i = PartFor(input_file, description_parts, description_all) |
| if i == -1: |
| raise SystemExit( |
| '%s is already in all parts, but matched ' |
| '%s in all_from_libs' % (input_file, obj)) |
| # See note in main(). |
| assert num_parts == 2, "Can't handle > 2 dlls currently" |
| by_parts[1 - i].append(obj) |
| match_count += 1 |
| if match_count == 0: |
| raise SystemExit( |
| '%s, %s matched a lib, but no objs' % (lib_spec, obj_spec)) |
| return by_parts |
| |
| |
| def main(): |
| flags, inputs, intermediate_manifest = GetFlagsAndInputs(sys.argv[1:]) |
| partition_file = os.path.normpath( |
| os.path.join(BASE_DIR, '../../../build/split_link_partition.py')) |
| with open(partition_file) as partition: |
| description = eval(partition.read()) |
| inputs_by_part = [] |
| description_parts = description['parts'] |
| # We currently assume that if a symbol isn't in dll 0, then it's in dll 1 |
| # when generating def files. Otherwise, we'd need to do more complex things |
| # to figure out where each symbol actually is to assign it to the correct |
| # .def file. |
| num_parts = len(description_parts) |
| assert num_parts == 2, "Can't handle > 2 dlls currently" |
| description_parts.reverse() |
| objs_from_libs = ExtractSubObjsTargetedAtAll( |
| inputs, |
| num_parts, |
| description_parts, |
| description['all'], |
| description['all_from_libs']) |
| objs_from_libs.reverse() |
| inputs_by_part = [[] for _ in range(num_parts)] |
| for input_file in inputs: |
| i = PartFor(input_file, description_parts, description['all']) |
| if i == -1: |
| for part in inputs_by_part: |
| part.append(input_file) |
| else: |
| inputs_by_part[i].append(input_file) |
| inputs_by_part.reverse() |
| |
| # Put the subobjs on to the main list. |
| for i, part in enumerate(objs_from_libs): |
| Log('%d sub .objs added to part %d' % (len(part), i)) |
| inputs_by_part[i].extend(part) |
| |
| unresolved_by_part = [[] for _ in range(num_parts)] |
| import_libs = [None] * num_parts |
| deffiles = [None] * num_parts |
| |
| data_exports = 0 |
| for i in range(5): |
| Log('--- starting pass %d' % i) |
| ok, dlls, unresolved_by_part = AttemptLink( |
| flags, inputs_by_part, unresolved_by_part, deffiles, import_libs, |
| intermediate_manifest) |
| if ok: |
| break |
| data_exports = 0 |
| for i, part in enumerate(unresolved_by_part): |
| for export in part: |
| if IsDataDefinition(export): |
| print 'part %d contains data export: %s (aka %s)' % ( |
| i, Unmangle(export), export) |
| data_exports += 1 |
| deffiles = GenerateDefFiles(unresolved_by_part) |
| import_libs = BuildImportLibs(flags, inputs_by_part, deffiles) |
| else: |
| if data_exports and not IGNORE_DATA: |
| print '%d data exports found, see report above.' % data_exports |
| print('These cannot be exported, and must be either duplicated to the ' |
| 'target DLL (if constant), or wrapped in a function.') |
| return 1 |
| |
| mt_exe = GetMtPath() |
| for i, dll in enumerate(dlls): |
| Log('embedding manifest in %s' % dll) |
| args = [mt_exe, '-nologo', '-manifest'] |
| args.append(ManifestNameForIndex(i)) |
| args.append(description['manifest']) |
| args.append('-outputresource:%s;2' % dll) |
| subprocess.check_call(args) |
| |
| Log('built %r' % dlls) |
| |
| return 0 |
| |
| |
| if __name__ == '__main__': |
| sys.exit(main()) |