|
"""Find modules used by a script, using introspection.""" |
|
|
|
from __future__ import generators |
|
import dis |
|
import imp |
|
import marshal |
|
import os |
|
import sys |
|
import types |
|
import struct |
|
|
|
if hasattr(sys.__stdout__, "newlines"): |
|
READ_MODE = "U" # universal line endings |
|
else: |
|
# Python < 2.3 compatibility, no longer strictly required |
|
READ_MODE = "r" |
|
|
|
LOAD_CONST = dis.opmap['LOAD_CONST'] |
|
IMPORT_NAME = dis.opmap['IMPORT_NAME'] |
|
STORE_NAME = dis.opmap['STORE_NAME'] |
|
STORE_GLOBAL = dis.opmap['STORE_GLOBAL'] |
|
STORE_OPS = STORE_NAME, STORE_GLOBAL |
|
HAVE_ARGUMENT = dis.HAVE_ARGUMENT |
|
EXTENDED_ARG = dis.EXTENDED_ARG |
|
|
|
def _unpack_opargs(code): |
|
# enumerate() is not an option, since we sometimes process |
|
# multiple elements on a single pass through the loop |
|
extended_arg = 0 |
|
n = len(code) |
|
i = 0 |
|
while i < n: |
|
op = ord(code[i]) |
|
offset = i |
|
i = i+1 |
|
arg = None |
|
if op >= HAVE_ARGUMENT: |
|
arg = ord(code[i]) + ord(code[i+1])*256 + extended_arg |
|
extended_arg = 0 |
|
i = i+2 |
|
if op == EXTENDED_ARG: |
|
extended_arg = arg*65536 |
|
yield (offset, op, arg) |
|
|
|
# Modulefinder does a good job at simulating Python's, but it can not |
|
# handle __path__ modifications packages make at runtime. Therefore there |
|
# is a mechanism whereby you can register extra paths in this map for a |
|
# package, and it will be honored. |
|
|
|
# Note this is a mapping is lists of paths. |
|
packagePathMap = {} |
|
|
|
# A Public interface |
|
def AddPackagePath(packagename, path): |
|
paths = packagePathMap.get(packagename, []) |
|
paths.append(path) |
|
packagePathMap[packagename] = paths |
|
|
|
replacePackageMap = {} |
|
|
|
# This ReplacePackage mechanism allows modulefinder to work around the |
|
# way the _xmlplus package injects itself under the name "xml" into |
|
# sys.modules at runtime by calling ReplacePackage("_xmlplus", "xml") |
|
# before running ModuleFinder. |
|
|
|
def ReplacePackage(oldname, newname): |
|
replacePackageMap[oldname] = newname |
|
|
|
|
|
class Module: |
|
|
|
def __init__(self, name, file=None, path=None): |
|
self.__name__ = name |
|
self.__file__ = file |
|
self.__path__ = path |
|
self.__code__ = None |
|
# The set of global names that are assigned to in the module. |
|
# This includes those names imported through starimports of |
|
# Python modules. |
|
self.globalnames = {} |
|
# The set of starimports this module did that could not be |
|
# resolved, ie. a starimport from a non-Python module. |
|
self.starimports = {} |
|
|
|
def __repr__(self): |
|
s = "Module(%r" % (self.__name__,) |
|
if self.__file__ is not None: |
|
s = s + ", %r" % (self.__file__,) |
|
if self.__path__ is not None: |
|
s = s + ", %r" % (self.__path__,) |
|
s = s + ")" |
|
return s |
|
|
|
class ModuleFinder: |
|
|
|
def __init__(self, path=None, debug=0, excludes=[], replace_paths=[]): |
|
if path is None: |
|
path = sys.path |
|
self.path = path |
|
self.modules = {} |
|
self.badmodules = {} |
|
self.debug = debug |
|
self.indent = 0 |
|
self.excludes = excludes |
|
self.replace_paths = replace_paths |
|
self.processed_paths = [] # Used in debugging only |
|
|
|
def msg(self, level, str, *args): |
|
if level <= self.debug: |
|
for i in range(self.indent): |
|
print " ", |
|
print str, |
|
for arg in args: |
|
print repr(arg), |
|
print |
|
|
|
def msgin(self, *args): |
|
level = args[0] |
|
if level <= self.debug: |
|
self.indent = self.indent + 1 |
|
self.msg(*args) |
|
|
|
def msgout(self, *args): |
|
level = args[0] |
|
if level <= self.debug: |
|
self.indent = self.indent - 1 |
|
self.msg(*args) |
|
|
|
def run_script(self, pathname): |
|
self.msg(2, "run_script", pathname) |
|
with open(pathname, READ_MODE) as fp: |
|
stuff = ("", "r", imp.PY_SOURCE) |
|
self.load_module('__main__', fp, pathname, stuff) |
|
|
|
def load_file(self, pathname): |
|
dir, name = os.path.split(pathname) |
|
name, ext = os.path.splitext(name) |
|
with open(pathname, READ_MODE) as fp: |
|
stuff = (ext, "r", imp.PY_SOURCE) |
|
self.load_module(name, fp, pathname, stuff) |
|
|
|
def import_hook(self, name, caller=None, fromlist=None, level=-1): |
|
self.msg(3, "import_hook", name, caller, fromlist, level) |
|
parent = self.determine_parent(caller, level=level) |
|
q, tail = self.find_head_package(parent, name) |
|
m = self.load_tail(q, tail) |
|
if not fromlist: |
|
return q |
|
if m.__path__: |
|
self.ensure_fromlist(m, fromlist) |
|
return None |
|
|
|
def determine_parent(self, caller, level=-1): |
|
self.msgin(4, "determine_parent", caller, level) |
|
if not caller or level == 0: |
|
self.msgout(4, "determine_parent -> None") |
|
return None |
|
pname = caller.__name__ |
|
if level >= 1: # relative import |
|
if caller.__path__: |
|
level -= 1 |
|
if level == 0: |
|
parent = self.modules[pname] |
|
assert parent is caller |
|
self.msgout(4, "determine_parent ->", parent) |
|
return parent |
|
if pname.count(".") < level: |
|
raise ImportError, "relative importpath too deep" |
|
pname = ".".join(pname.split(".")[:-level]) |
|
parent = self.modules[pname] |
|
self.msgout(4, "determine_parent ->", parent) |
|
return parent |
|
if caller.__path__: |
|
parent = self.modules[pname] |
|
assert caller is parent |
|
self.msgout(4, "determine_parent ->", parent) |
|
return parent |
|
if '.' in pname: |
|
i = pname.rfind('.') |
|
pname = pname[:i] |
|
parent = self.modules[pname] |
|
assert parent.__name__ == pname |
|
self.msgout(4, "determine_parent ->", parent) |
|
return parent |
|
self.msgout(4, "determine_parent -> None") |
|
return None |
|
|
|
def find_head_package(self, parent, name): |
|
self.msgin(4, "find_head_package", parent, name) |
|
if '.' in name: |
|
i = name.find('.') |
|
head = name[:i] |
|
tail = name[i+1:] |
|
else: |
|
head = name |
|
tail = "" |
|
if parent: |
|
qname = "%s.%s" % (parent.__name__, head) |
|
else: |
|
qname = head |
|
q = self.import_module(head, qname, parent) |
|
if q: |
|
self.msgout(4, "find_head_package ->", (q, tail)) |
|
return q, tail |
|
if parent: |
|
qname = head |
|
parent = None |
|
q = self.import_module(head, qname, parent) |
|
if q: |
|
self.msgout(4, "find_head_package ->", (q, tail)) |
|
return q, tail |
|
self.msgout(4, "raise ImportError: No module named", qname) |
|
raise ImportError, "No module named " + qname |
|
|
|
def load_tail(self, q, tail): |
|
self.msgin(4, "load_tail", q, tail) |
|
m = q |
|
while tail: |
|
i = tail.find('.') |
|
if i < 0: i = len(tail) |
|
head, tail = tail[:i], tail[i+1:] |
|
mname = "%s.%s" % (m.__name__, head) |
|
m = self.import_module(head, mname, m) |
|
if not m: |
|
self.msgout(4, "raise ImportError: No module named", mname) |
|
raise ImportError, "No module named " + mname |
|
self.msgout(4, "load_tail ->", m) |
|
return m |
|
|
|
def ensure_fromlist(self, m, fromlist, recursive=0): |
|
self.msg(4, "ensure_fromlist", m, fromlist, recursive) |
|
for sub in fromlist: |
|
if sub == "*": |
|
if not recursive: |
|
all = self.find_all_submodules(m) |
|
if all: |
|
self.ensure_fromlist(m, all, 1) |
|
elif not hasattr(m, sub): |
|
subname = "%s.%s" % (m.__name__, sub) |
|
submod = self.import_module(sub, subname, m) |
|
if not submod: |
|
raise ImportError, "No module named " + subname |
|
|
|
def find_all_submodules(self, m): |
|
if not m.__path__: |
|
return |
|
modules = {} |
|
# 'suffixes' used to be a list hardcoded to [".py", ".pyc", ".pyo"]. |
|
# But we must also collect Python extension modules - although |
|
# we cannot separate normal dlls from Python extensions. |
|
suffixes = [] |
|
for triple in imp.get_suffixes(): |
|
suffixes.append(triple[0]) |
|
for dir in m.__path__: |
|
try: |
|
names = os.listdir(dir) |
|
except os.error: |
|
self.msg(2, "can't list directory", dir) |
|
continue |
|
for name in names: |
|
mod = None |
|
for suff in suffixes: |
|
n = len(suff) |
|
if name[-n:] == suff: |
|
mod = name[:-n] |
|
break |
|
if mod and mod != "__init__": |
|
modules[mod] = mod |
|
return modules.keys() |
|
|
|
def import_module(self, partname, fqname, parent): |
|
self.msgin(3, "import_module", partname, fqname, parent) |
|
try: |
|
m = self.modules[fqname] |
|
except KeyError: |
|
pass |
|
else: |
|
self.msgout(3, "import_module ->", m) |
|
return m |
|
if fqname in self.badmodules: |
|
self.msgout(3, "import_module -> None") |
|
return None |
|
if parent and parent.__path__ is None: |
|
self.msgout(3, "import_module -> None") |
|
return None |
|
try: |
|
fp, pathname, stuff = self.find_module(partname, |
|
parent and parent.__path__, parent) |
|
except ImportError: |
|
self.msgout(3, "import_module ->", None) |
|
return None |
|
try: |
|
m = self.load_module(fqname, fp, pathname, stuff) |
|
finally: |
|
if fp: fp.close() |
|
if parent: |
|
setattr(parent, partname, m) |
|
self.msgout(3, "import_module ->", m) |
|
return m |
|
|
|
def load_module(self, fqname, fp, pathname, file_info): |
|
suffix, mode, type = file_info |
|
self.msgin(2, "load_module", fqname, fp and "fp", pathname) |
|
if type == imp.PKG_DIRECTORY: |
|
m = self.load_package(fqname, pathname) |
|
self.msgout(2, "load_module ->", m) |
|
return m |
|
if type == imp.PY_SOURCE: |
|
co = compile(fp.read()+'\n', pathname, 'exec') |
|
elif type == imp.PY_COMPILED: |
|
if fp.read(4) != imp.get_magic(): |
|
self.msgout(2, "raise ImportError: Bad magic number", pathname) |
|
raise ImportError, "Bad magic number in %s" % pathname |
|
fp.read(4) |
|
co = marshal.load(fp) |
|
else: |
|
co = None |
|
m = self.add_module(fqname) |
|
m.__file__ = pathname |
|
if co: |
|
if self.replace_paths: |
|
co = self.replace_paths_in_code(co) |
|
m.__code__ = co |
|
self.scan_code(co, m) |
|
self.msgout(2, "load_module ->", m) |
|
return m |
|
|
|
def _add_badmodule(self, name, caller): |
|
if name not in self.badmodules: |
|
self.badmodules[name] = {} |
|
if caller: |
|
self.badmodules[name][caller.__name__] = 1 |
|
else: |
|
self.badmodules[name]["-"] = 1 |
|
|
|
def _safe_import_hook(self, name, caller, fromlist, level=-1): |
|
# wrapper for self.import_hook() that won't raise ImportError |
|
if name in self.badmodules: |
|
self._add_badmodule(name, caller) |
|
return |
|
try: |
|
self.import_hook(name, caller, level=level) |
|
except ImportError, msg: |
|
self.msg(2, "ImportError:", str(msg)) |
|
self._add_badmodule(name, caller) |
|
else: |
|
if fromlist: |
|
for sub in fromlist: |
|
if sub in self.badmodules: |
|
self._add_badmodule(sub, caller) |
|
continue |
|
try: |
|
self.import_hook(name, caller, [sub], level=level) |
|
except ImportError, msg: |
|
self.msg(2, "ImportError:", str(msg)) |
|
fullname = name + "." + sub |
|
self._add_badmodule(fullname, caller) |
|
|
|
def scan_opcodes(self, co, |
|
unpack = struct.unpack): |
|
# Scan the code, and yield 'interesting' opcode combinations |
|
# Version for Python 2.4 and older |
|
code = co.co_code |
|
names = co.co_names |
|
consts = co.co_consts |
|
opargs = [(op, arg) for _, op, arg in _unpack_opargs(code) |
|
if op != EXTENDED_ARG] |
|
for i, (op, oparg) in enumerate(opargs): |
|
if c in STORE_OPS: |
|
yield "store", (names[oparg],) |
|
continue |
|
if (op == IMPORT_NAME and i >= 1 |
|
and opargs[i-1][0] == LOAD_CONST): |
|
fromlist = consts[opargs[i-1][1]] |
|
yield "import", (fromlist, names[oparg]) |
|
continue |
|
|
|
def scan_opcodes_25(self, co): |
|
# Scan the code, and yield 'interesting' opcode combinations |
|
code = co.co_code |
|
names = co.co_names |
|
consts = co.co_consts |
|
opargs = [(op, arg) for _, op, arg in _unpack_opargs(code) |
|
if op != EXTENDED_ARG] |
|
for i, (op, oparg) in enumerate(opargs): |
|
if op in STORE_OPS: |
|
yield "store", (names[oparg],) |
|
continue |
|
if (op == IMPORT_NAME and i >= 2 |
|
and opargs[i-1][0] == opargs[i-2][0] == LOAD_CONST): |
|
level = consts[opargs[i-2][1]] |
|
fromlist = consts[opargs[i-1][1]] |
|
if level == -1: # normal import |
|
yield "import", (fromlist, names[oparg]) |
|
elif level == 0: # absolute import |
|
yield "absolute_import", (fromlist, names[oparg]) |
|
else: # relative import |
|
yield "relative_import", (level, fromlist, names[oparg]) |
|
continue |
|
|
|
def scan_code(self, co, m): |
|
code = co.co_code |
|
if sys.version_info >= (2, 5): |
|
scanner = self.scan_opcodes_25 |
|
else: |
|
scanner = self.scan_opcodes |
|
for what, args in scanner(co): |
|
if what == "store": |
|
name, = args |
|
m.globalnames[name] = 1 |
|
elif what in ("import", "absolute_import"): |
|
fromlist, name = args |
|
have_star = 0 |
|
if fromlist is not None: |
|
if "*" in fromlist: |
|
have_star = 1 |
|
fromlist = [f for f in fromlist if f != "*"] |
|
if what == "absolute_import": level = 0 |
|
else: level = -1 |
|
self._safe_import_hook(name, m, fromlist, level=level) |
|
if have_star: |
|
# We've encountered an "import *". If it is a Python module, |
|
# the code has already been parsed and we can suck out the |
|
# global names. |
|
mm = None |
|
if m.__path__: |
|
# At this point we don't know whether 'name' is a |
|
# submodule of 'm' or a global module. Let's just try |
|
# the full name first. |
|
mm = self.modules.get(m.__name__ + "." + name) |
|
if mm is None: |
|
mm = self.modules.get(name) |
|
if mm is not None: |
|
m.globalnames.update(mm.globalnames) |
|
m.starimports.update(mm.starimports) |
|
if mm.__code__ is None: |
|
m.starimports[name] = 1 |
|
else: |
|
m.starimports[name] = 1 |
|
elif what == "relative_import": |
|
level, fromlist, name = args |
|
if name: |
|
self._safe_import_hook(name, m, fromlist, level=level) |
|
else: |
|
parent = self.determine_parent(m, level=level) |
|
self._safe_import_hook(parent.__name__, None, fromlist, level=0) |
|
else: |
|
# We don't expect anything else from the generator. |
|
raise RuntimeError(what) |
|
|
|
for c in co.co_consts: |
|
if isinstance(c, type(co)): |
|
self.scan_code(c, m) |
|
|
|
def load_package(self, fqname, pathname): |
|
self.msgin(2, "load_package", fqname, pathname) |
|
newname = replacePackageMap.get(fqname) |
|
if newname: |
|
fqname = newname |
|
m = self.add_module(fqname) |
|
m.__file__ = pathname |
|
m.__path__ = [pathname] |
|
|
|
# As per comment at top of file, simulate runtime __path__ additions. |
|
m.__path__ = m.__path__ + packagePathMap.get(fqname, []) |
|
|
|
fp, buf, stuff = self.find_module("__init__", m.__path__) |
|
self.load_module(fqname, fp, buf, stuff) |
|
self.msgout(2, "load_package ->", m) |
|
if fp: |
|
fp.close() |
|
return m |
|
|
|
def add_module(self, fqname): |
|
if fqname in self.modules: |
|
return self.modules[fqname] |
|
self.modules[fqname] = m = Module(fqname) |
|
return m |
|
|
|
def find_module(self, name, path, parent=None): |
|
if parent is not None: |
|
# assert path is not None |
|
fullname = parent.__name__+'.'+name |
|
else: |
|
fullname = name |
|
if fullname in self.excludes: |
|
self.msgout(3, "find_module -> Excluded", fullname) |
|
raise ImportError, name |
|
|
|
if path is None: |
|
if name in sys.builtin_module_names: |
|
return (None, None, ("", "", imp.C_BUILTIN)) |
|
|
|
path = self.path |
|
return imp.find_module(name, path) |
|
|
|
def report(self): |
|
"""Print a report to stdout, listing the found modules with their |
|
paths, as well as modules that are missing, or seem to be missing. |
|
""" |
|
print |
|
print " %-25s %s" % ("Name", "File") |
|
print " %-25s %s" % ("----", "----") |
|
# Print modules found |
|
keys = self.modules.keys() |
|
keys.sort() |
|
for key in keys: |
|
m = self.modules[key] |
|
if m.__path__: |
|
print "P", |
|
else: |
|
print "m", |
|
print "%-25s" % key, m.__file__ or "" |
|
|
|
# Print missing modules |
|
missing, maybe = self.any_missing_maybe() |
|
if missing: |
|
print |
|
print "Missing modules:" |
|
for name in missing: |
|
mods = self.badmodules[name].keys() |
|
mods.sort() |
|
print "?", name, "imported from", ', '.join(mods) |
|
# Print modules that may be missing, but then again, maybe not... |
|
if maybe: |
|
print |
|
print "Submodules that appear to be missing, but could also be", |
|
print "global names in the parent package:" |
|
for name in maybe: |
|
mods = self.badmodules[name].keys() |
|
mods.sort() |
|
print "?", name, "imported from", ', '.join(mods) |
|
|
|
def any_missing(self): |
|
"""Return a list of modules that appear to be missing. Use |
|
any_missing_maybe() if you want to know which modules are |
|
certain to be missing, and which *may* be missing. |
|
""" |
|
missing, maybe = self.any_missing_maybe() |
|
return missing + maybe |
|
|
|
def any_missing_maybe(self): |
|
"""Return two lists, one with modules that are certainly missing |
|
and one with modules that *may* be missing. The latter names could |
|
either be submodules *or* just global names in the package. |
|
|
|
The reason it can't always be determined is that it's impossible to |
|
tell which names are imported when "from module import *" is done |
|
with an extension module, short of actually importing it. |
|
""" |
|
missing = [] |
|
maybe = [] |
|
for name in self.badmodules: |
|
if name in self.excludes: |
|
continue |
|
i = name.rfind(".") |
|
if i < 0: |
|
missing.append(name) |
|
continue |
|
subname = name[i+1:] |
|
pkgname = name[:i] |
|
pkg = self.modules.get(pkgname) |
|
if pkg is not None: |
|
if pkgname in self.badmodules[name]: |
|
# The package tried to import this module itself and |
|
# failed. It's definitely missing. |
|
missing.append(name) |
|
elif subname in pkg.globalnames: |
|
# It's a global in the package: definitely not missing. |
|
pass |
|
elif pkg.starimports: |
|
# It could be missing, but the package did an "import *" |
|
# from a non-Python module, so we simply can't be sure. |
|
maybe.append(name) |
|
else: |
|
# It's not a global in the package, the package didn't |
|
# do funny star imports, it's very likely to be missing. |
|
# The symbol could be inserted into the package from the |
|
# outside, but since that's not good style we simply list |
|
# it missing. |
|
missing.append(name) |
|
else: |
|
missing.append(name) |
|
missing.sort() |
|
maybe.sort() |
|
return missing, maybe |
|
|
|
def replace_paths_in_code(self, co): |
|
new_filename = original_filename = os.path.normpath(co.co_filename) |
|
for f, r in self.replace_paths: |
|
if original_filename.startswith(f): |
|
new_filename = r + original_filename[len(f):] |
|
break |
|
|
|
if self.debug and original_filename not in self.processed_paths: |
|
if new_filename != original_filename: |
|
self.msgout(2, "co_filename %r changed to %r" \ |
|
% (original_filename,new_filename,)) |
|
else: |
|
self.msgout(2, "co_filename %r remains unchanged" \ |
|
% (original_filename,)) |
|
self.processed_paths.append(original_filename) |
|
|
|
consts = list(co.co_consts) |
|
for i in range(len(consts)): |
|
if isinstance(consts[i], type(co)): |
|
consts[i] = self.replace_paths_in_code(consts[i]) |
|
|
|
return types.CodeType(co.co_argcount, co.co_nlocals, co.co_stacksize, |
|
co.co_flags, co.co_code, tuple(consts), co.co_names, |
|
co.co_varnames, new_filename, co.co_name, |
|
co.co_firstlineno, co.co_lnotab, |
|
co.co_freevars, co.co_cellvars) |
|
|
|
|
|
def test(): |
|
# Parse command line |
|
import getopt |
|
try: |
|
opts, args = getopt.getopt(sys.argv[1:], "dmp:qx:") |
|
except getopt.error, msg: |
|
print msg |
|
return |
|
|
|
# Process options |
|
debug = 1 |
|
domods = 0 |
|
addpath = [] |
|
exclude = [] |
|
for o, a in opts: |
|
if o == '-d': |
|
debug = debug + 1 |
|
if o == '-m': |
|
domods = 1 |
|
if o == '-p': |
|
addpath = addpath + a.split(os.pathsep) |
|
if o == '-q': |
|
debug = 0 |
|
if o == '-x': |
|
exclude.append(a) |
|
|
|
# Provide default arguments |
|
if not args: |
|
script = "hello.py" |
|
else: |
|
script = args[0] |
|
|
|
# Set the path based on sys.path and the script directory |
|
path = sys.path[:] |
|
path[0] = os.path.dirname(script) |
|
path = addpath + path |
|
if debug > 1: |
|
print "path:" |
|
for item in path: |
|
print " ", repr(item) |
|
|
|
# Create the module finder and turn its crank |
|
mf = ModuleFinder(path, debug, exclude) |
|
for arg in args[1:]: |
|
if arg == '-m': |
|
domods = 1 |
|
continue |
|
if domods: |
|
if arg[-2:] == '.*': |
|
mf.import_hook(arg[:-2], None, ["*"]) |
|
else: |
|
mf.import_hook(arg) |
|
else: |
|
mf.load_file(arg) |
|
mf.run_script(script) |
|
mf.report() |
|
return mf # for -i debugging |
|
|
|
|
|
if __name__ == '__main__': |
|
try: |
|
mf = test() |
|
except KeyboardInterrupt: |
|
print "\n[interrupt]" |
