| from generator3.core import OriginType | |
| from generator3.util_methods import * | |
| from generator3.util_methods import get_relative_path_by_qname | |
| from generator3.docstring_parsing import * | |
| class emptylistdict(dict): | |
| """defaultdict not available before 2.5; simplest reimplementation using [] as default""" | |
| def __getitem__(self, item): | |
| if item in self: | |
| return dict.__getitem__(self, item) | |
| else: | |
| it = [] | |
| self.__setitem__(item, it) | |
| return it | |
| class Buf(object): | |
| """Buffers data in a list, can write to a file. Indentation is provided externally.""" | |
| def __init__(self, indenter): | |
| self.data = [] | |
| self.indenter = indenter | |
| def put(self, data): | |
| if data: | |
| self.data.append(ensureUnicode(data)) | |
| def out(self, indent, *what): | |
| """Output the arguments, indenting as needed, and adding an eol""" | |
| self.put(self.indenter.indent(indent)) | |
| for item in what: | |
| self.put(item) | |
| self.put("\n") | |
| def flush_bytes(self, outfile): | |
| for data in self.data: | |
| outfile.write(data.encode(OUT_ENCODING, "replace")) | |
| def flush_str(self, outfile): | |
| for data in self.data: | |
| outfile.write(data) | |
| if version[0] < 3: | |
| flush = flush_bytes | |
| else: | |
| flush = flush_str | |
| def isEmpty(self): | |
| return len(self.data) == 0 | |
| class ClassBuf(Buf): | |
| def __init__(self, name, indenter): | |
| super(ClassBuf, self).__init__(indenter) | |
| self.name = name | |
| #noinspection PyBroadException | |
| class ModuleRedeclarator(object): | |
| def __init__(self, module, mod_qname, mod_filename, cache_dir, indent_size=4, doing_builtins=False): | |
| """ | |
| @param module: module object | |
| @param mod_qname: module qualified name | |
| @param mod_filename: filename of binary module (the .dll or .so). Can be None for modules | |
| that don't have corresponding binary files (e.g. builtins) | |
| @param cache_dir: per-binary cache directory where the generated stub will be stored. | |
| Normally, it's "<IDE system dir>/python_stubs/cache/<sha256 digest>/". | |
| @param indent_size: amount of space characters per indent | |
| """ | |
| import generator3.core | |
| self.test_mode = generator3.core.is_test_mode() | |
| self.gen_version = generator3.core.version() | |
| self.module = module | |
| self.qname = mod_qname | |
| self.cache_dir = cache_dir | |
| self.mod_filename = mod_filename | |
| # we write things into buffers out-of-order | |
| self.header_buf = Buf(self) | |
| self.imports_buf = Buf(self) | |
| self.functions_buf = Buf(self) | |
| self.classes_buf = Buf(self) | |
| self.classes_buffs = list() | |
| self.footer_buf = Buf(self) | |
| self.indent_size = indent_size | |
| self._indent_step = " " * self.indent_size | |
| self.split_modules = False | |
| # | |
| self.imported_modules = {"": the_builtins} # explicit module imports: {"name": module} | |
| self.hidden_imports = {} # {'real_mod_name': 'alias'}; we alias names with "__" since we don't want them exported | |
| # ^ used for things that we don't re-export but need to import, e.g. certain base classes in gnome. | |
| self._defined = {} # stores True for every name defined so far, to break circular refs in values | |
| self.doing_builtins = doing_builtins | |
| self.ret_type_cache = {} | |
| self.used_imports = emptylistdict() # qual_mod_name -> [imported_names,..]: actually used imported names | |
| def _initializeQApp4(self): | |
| try: # QtGui should be imported _before_ QtCore package. | |
| # This is done for the QWidget references from QtCore (such as QSignalMapper). Known bug in PyQt 4.7+ | |
| # Causes "TypeError: C++ type 'QWidget*' is not supported as a native Qt signal type" | |
| import PyQt4.QtGui | |
| except ImportError: | |
| pass | |
| # manually instantiate and keep reference to singleton QCoreApplication (we don't want it to be deleted during the introspection) | |
| # use QCoreApplication instead of QApplication to avoid blinking app in Dock on Mac OS | |
| try: | |
| from PyQt4.QtCore import QCoreApplication | |
| self.app = QCoreApplication([]) | |
| return | |
| except ImportError: | |
| pass | |
| def _initializeQApp5(self): | |
| try: | |
| from PyQt5.QtCore import QCoreApplication | |
| self.app = QCoreApplication([]) | |
| return | |
| except ImportError: | |
| pass | |
| def indent(self, level): | |
| """Return indentation whitespace for given level.""" | |
| return self._indent_step * level | |
| def flush(self): | |
| qname_parts = self.qname.split('.') | |
| if self.split_modules: | |
| last_pkg_dir = build_pkg_structure(self.cache_dir, self.qname) | |
| with fopen(os.path.join(last_pkg_dir, "__init__.py"), "w") as init: | |
| for buf in (self.header_buf, self.imports_buf, self.functions_buf, self.classes_buf): | |
| buf.flush(init) | |
| data = "" | |
| for buf in self.classes_buffs: | |
| with fopen(os.path.join(last_pkg_dir, buf.name) + '.py', "w") as dummy: | |
| self.header_buf.flush(dummy) | |
| self.imports_buf.flush(dummy) | |
| buf.flush(dummy) | |
| data += self.create_local_import(buf.name) | |
| init.write(data) | |
| self.footer_buf.flush(init) | |
| else: | |
| last_pkg_dir = build_pkg_structure(self.cache_dir, '.'.join(qname_parts[:-1])) | |
| # In some rare cases submodules of a binary might have been generated earlier than the module | |
| # for the binary itself. For instance, it happens for "pyexpat" built-in module which | |
| # submodules "pyexpat.errors" and "pyexpat.model" are processed together with "_elementtree" | |
| # and "pickle" before "pyexpat" and thus empty pyexpat/__init__.py for them should be replaced | |
| # with the skeleton for the main module itself later on. | |
| existing_pkg_init = os.path.join(last_pkg_dir, qname_parts[-1], '__init__.py') | |
| if os.path.exists(existing_pkg_init): | |
| skeleton_path = existing_pkg_init | |
| else: | |
| skeleton_path = os.path.join(last_pkg_dir, qname_parts[-1] + '.py') | |
| with fopen(skeleton_path, "w") as mod: | |
| for buf in (self.header_buf, self.imports_buf, self.functions_buf, self.classes_buf): | |
| buf.flush(mod) | |
| for buf in self.classes_buffs: | |
| buf.flush(mod) | |
| self.footer_buf.flush(mod) | |
| # Some builtin classes effectively change __init__ signature without overriding it. | |
| # This callable serves as a placeholder to be replaced via REDEFINED_BUILTIN_SIGS | |
| def fake_builtin_init(self): | |
| pass # just a callable, sig doesn't matter | |
| fake_builtin_init.__doc__ = object.__init__.__doc__ # this forces class's doc to be used instead | |
| def create_local_import(self, name): | |
| if len(name.split(".")) > 1: return "" | |
| data = "from " | |
| if version[0] >= 3: | |
| data += "." | |
| data += name + " import " + name + "\n" | |
| return data | |
| def find_imported_name(self, item): | |
| """ | |
| Finds out how the item is represented in imported modules. | |
| @param item what to check | |
| @return qualified name (like "sys.stdin") or None | |
| """ | |
| # TODO: return a pair, not a glued string | |
| if not isinstance(item, SIMPLEST_TYPES): | |
| for mname in self.imported_modules: | |
| m = self.imported_modules[mname] | |
| for inner_name in m.__dict__: | |
| suspect = getattr(m, inner_name) | |
| if suspect is item: | |
| if mname: | |
| mname += "." | |
| elif self.module is the_builtins: # don't short-circuit builtins | |
| return None | |
| return mname + inner_name | |
| return None | |
| _initializers = ( | |
| (dict, "{}"), | |
| (tuple, "()"), | |
| (list, "[]"), | |
| ) | |
| def invent_initializer(self, a_type): | |
| """ | |
| Returns an innocuous initializer expression for a_type, or "None" | |
| """ | |
| for initializer_type, r in self._initializers: | |
| if initializer_type == a_type: | |
| return r | |
| # NOTE: here we could handle things like defaultdict, sets, etc if we wanted | |
| return "None" | |
| def fmt_value(self, out, p_value, indent, prefix="", postfix="", as_name=None, seen_values=None): | |
| """ | |
| Formats and outputs value (it occupies an entire line or several lines). | |
| @param out function that does output (a Buf.out) | |
| @param p_value the value. | |
| @param indent indent level. | |
| @param prefix text to print before the value | |
| @param postfix text to print after the value | |
| @param as_name hints which name are we trying to print; helps with circular refs. | |
| @param seen_values a list of keys we've seen if we're processing a dict | |
| """ | |
| SELF_VALUE = "<value is a self-reference, replaced by this string>" | |
| ERR_VALUE = "<failed to retrieve the value>" | |
| if isinstance(p_value, SIMPLEST_TYPES): | |
| out(indent, prefix, reliable_repr(p_value), postfix) | |
| else: | |
| if sys.platform == "cli": | |
| imported_name = None | |
| else: | |
| imported_name = self.find_imported_name(p_value) | |
| if imported_name: | |
| out(indent, prefix, imported_name, postfix) | |
| # TODO: kind of self.used_imports[imported_name].append(p_value) but split imported_name | |
| # else we could potentially return smth we did not otherwise import. but not likely. | |
| else: | |
| if isinstance(p_value, (list, tuple)): | |
| if not seen_values: | |
| seen_values = [p_value] | |
| if len(p_value) == 0: | |
| out(indent, prefix, repr(p_value), postfix) | |
| else: | |
| if isinstance(p_value, list): | |
| lpar, rpar = "[", "]" | |
| else: | |
| lpar, rpar = "(", ")" | |
| out(indent, prefix, lpar) | |
| for value in p_value: | |
| if value in seen_values: | |
| value = SELF_VALUE | |
| elif not isinstance(value, SIMPLEST_TYPES): | |
| seen_values.append(value) | |
| self.fmt_value(out, value, indent + 1, postfix=",", seen_values=seen_values) | |
| out(indent, rpar, postfix) | |
| elif isinstance(p_value, dict): | |
| if len(p_value) == 0: | |
| out(indent, prefix, repr(p_value), postfix) | |
| else: | |
| if not seen_values: | |
| seen_values = [p_value] | |
| out(indent, prefix, "{") | |
| keys = list(p_value.keys()) | |
| try: | |
| keys.sort() | |
| except TypeError: | |
| pass # unsortable keys happen, e,g, in py3k _ctypes | |
| for k in keys: | |
| value = p_value[k] | |
| try: | |
| is_seen = value in seen_values | |
| except: | |
| is_seen = False | |
| value = ERR_VALUE | |
| if is_seen: | |
| value = SELF_VALUE | |
| elif not isinstance(value, SIMPLEST_TYPES): | |
| seen_values.append(value) | |
| if isinstance(k, SIMPLEST_TYPES): | |
| self.fmt_value(out, value, indent + 1, prefix=repr(k) + ": ", postfix=",", | |
| seen_values=seen_values) | |
| else: | |
| # both key and value need fancy formatting | |
| self.fmt_value(out, k, indent + 1, postfix=": ", seen_values=seen_values) | |
| self.fmt_value(out, value, indent + 2, seen_values=seen_values) | |
| out(indent + 1, ",") | |
| out(indent, "}", postfix) | |
| else: # something else, maybe representable | |
| # look up this value in the module. | |
| if sys.platform == "cli": | |
| out(indent, prefix, "None", postfix) | |
| return | |
| found_name = "" | |
| for inner_name in self.module.__dict__: | |
| if self.module.__dict__[inner_name] is p_value: | |
| found_name = inner_name | |
| break | |
| if self._defined.get(found_name, False): | |
| out(indent, prefix, found_name, postfix) | |
| elif hasattr(self, "app"): | |
| return | |
| else: | |
| # a forward / circular declaration happens | |
| notice = "" | |
| try: | |
| representation = repr(p_value) | |
| except Exception: | |
| import traceback | |
| traceback.print_exc(file=sys.stderr) | |
| return | |
| if not self.test_mode: | |
| real_value = cleanup(representation) | |
| else: | |
| # Don't rely on repr() output in tests, as it may contain memory layout dependent id | |
| real_value = '' | |
| if found_name: | |
| if found_name == as_name: | |
| notice = " # (!) real value is %r" % real_value | |
| real_value = "None" | |
| else: | |
| notice = " # (!) forward: %s, real value is %r" % (found_name, real_value) | |
| if SANE_REPR_RE.match(real_value) and is_valid_expr(real_value): | |
| out(indent, prefix, real_value, postfix, notice) | |
| else: | |
| if not found_name: | |
| notice = " # (!) real value is %r" % real_value | |
| out(indent, prefix, "None", postfix, notice) | |
| def get_ret_type(self, attr): | |
| """ | |
| Returns a return type string as given by T_RETURN in tokens, or None | |
| """ | |
| if attr: | |
| ret_type = RET_TYPE.get(attr, None) | |
| if ret_type: | |
| return ret_type | |
| thing = getattr(self.module, attr, None) | |
| if thing: | |
| if not isinstance(thing, type) and is_callable(thing): # a function | |
| return None # TODO: maybe divinate a return type; see pygame.mixer.Channel | |
| return attr | |
| # adds no noticeable slowdown, I did measure. dch. | |
| for im_name, im_module in self.imported_modules.items(): | |
| cache_key = (im_name, attr) | |
| cached = self.ret_type_cache.get(cache_key, None) | |
| if cached: | |
| return cached | |
| ret_type = getattr(im_module, attr, None) | |
| if ret_type: | |
| if isinstance(ret_type, type): | |
| # detect a constructor | |
| constr_args = detect_constructor(ret_type) | |
| if constr_args is None: | |
| constr_args = "*(), **{}" # a silly catch-all constructor | |
| reference = "%s(%s)" % (attr, constr_args) | |
| elif is_callable(ret_type): # a function, classes are ruled out above | |
| return None | |
| else: | |
| reference = attr | |
| if im_name: | |
| result = "%s.%s" % (im_name, reference) | |
| else: # built-in | |
| result = reference | |
| self.ret_type_cache[cache_key] = result | |
| return result | |
| # TODO: handle things like "[a, b,..] and (foo,..)" | |
| return None | |
| SIG_DOC_NOTE = "restored from __doc__" | |
| SIG_DOC_UNRELIABLY = "NOTE: unreliably restored from __doc__ " | |
| def restore_by_docstring(self, signature_string, class_name, deco=None, ret_hint=None): | |
| """ | |
| @param signature_string: parameter list extracted from the doc string. | |
| @param class_name: name of the containing class, or None | |
| @param deco: decorator to use | |
| @param ret_hint: return type hint, if available | |
| @return (reconstructed_spec, return_type, note) or (None, _, _) if failed. | |
| """ | |
| action("restoring func %r of class %r", signature_string, class_name) | |
| # parse | |
| parsing_failed = False | |
| ret_type = None | |
| try: | |
| # strict parsing | |
| tokens = paramSeqAndRest.parseString(signature_string, True) | |
| ret_name = None | |
| if tokens: | |
| ret_t = tokens[-1] | |
| if ret_t[0] is T_RETURN: | |
| ret_name = ret_t[1] | |
| ret_type = self.get_ret_type(ret_name) or self.get_ret_type(ret_hint) | |
| except ParseException: | |
| # it did not parse completely; scavenge what we can | |
| parsing_failed = True | |
| tokens = [] | |
| try: | |
| # most unrestrictive parsing | |
| tokens = paramSeq.parseString(signature_string, False) | |
| except ParseException: | |
| pass | |
| # | |
| seq = transform_seq(tokens) | |
| # add safe defaults for unparsed | |
| if parsing_failed: | |
| doc_node = self.SIG_DOC_UNRELIABLY | |
| starred = None | |
| double_starred = None | |
| for one in seq: | |
| if type(one) is str: | |
| if one.startswith("**"): | |
| double_starred = one | |
| elif one.startswith("*"): | |
| starred = one | |
| if not starred: | |
| seq.append("*args") | |
| if not double_starred: | |
| seq.append("**kwargs") | |
| else: | |
| doc_node = self.SIG_DOC_NOTE | |
| # add 'self' if needed YYY | |
| if class_name and (not seq or seq[0] != 'self'): | |
| first_param = propose_first_param(deco) | |
| if first_param: | |
| seq.insert(0, first_param) | |
| seq = make_names_unique(seq) | |
| return (seq, ret_type, doc_node) | |
| def parse_func_doc(self, func_doc, func_id, func_name, class_name, deco=None, sip_generated=False): | |
| """ | |
| @param func_doc: __doc__ of the function. | |
| @param func_id: name to look for as identifier of the function in docstring | |
| @param func_name: name of the function. | |
| @param class_name: name of the containing class, or None | |
| @param deco: decorator to use | |
| @return (reconstructed_spec, return_literal, note) or (None, _, _) if failed. | |
| """ | |
| if sip_generated: | |
| overloads = [] | |
| for part in func_doc.split('\n'): | |
| signature = func_id + '(' | |
| i = part.find(signature) | |
| if i >= 0: | |
| overloads.append(part[i + len(signature):]) | |
| if len(overloads) > 1: | |
| docstring_results = [self.restore_by_docstring(overload, class_name, deco) for overload in overloads] | |
| ret_types = [] | |
| for result in docstring_results: | |
| rt = result[1] | |
| if rt and rt not in ret_types: | |
| ret_types.append(rt) | |
| if ret_types: | |
| ret_literal = " or ".join(ret_types) | |
| else: | |
| ret_literal = None | |
| param_lists = [result[0] for result in docstring_results] | |
| spec = build_signature(func_name, restore_parameters_for_overloads(param_lists)) | |
| return (spec, ret_literal, "restored from __doc__ with multiple overloads") | |
| # find the first thing to look like a definition | |
| prefix_re = re.compile(r"\s*(?:(\w+)[ \t]+)?" + func_id + r"\s*\(") # "foo(..." or "int foo(..." | |
| match = prefix_re.search(func_doc) # Note: this and previous line may consume up to 35% of time | |
| # parse the part that looks right | |
| if match: | |
| ret_hint = match.group(1) | |
| params, ret_literal, doc_note = self.restore_by_docstring(func_doc[match.end():], class_name, deco, ret_hint) | |
| spec = func_name + flatten(params) | |
| return (spec, ret_literal, doc_note) | |
| else: | |
| return (None, None, None) | |
| def is_predefined_builtin(self, module_name, class_name, func_name): | |
| return self.doing_builtins and module_name == BUILTIN_MOD_NAME and ( | |
| class_name, func_name) in PREDEFINED_BUILTIN_SIGS | |
| def redo_function(self, out, p_func, p_name, indent, p_class=None, p_modname=None, classname=None, seen=None): | |
| """ | |
| Restore function argument list as best we can. | |
| @param out output function of a Buf | |
| @param p_func function or method object | |
| @param p_name function name as known to owner | |
| @param indent indentation level | |
| @param p_class the class that contains this function as a method | |
| @param p_modname module name | |
| @param seen {id(func): name} map of functions already seen in the same namespace; | |
| id() because *some* functions are unhashable (eg _elementtree.Comment in py2.7) | |
| """ | |
| action("redoing func %r of class %r", p_name, p_class) | |
| if seen is not None: | |
| other_func = seen.get(id(p_func), None) | |
| if other_func and getattr(other_func, "__doc__", None) is getattr(p_func, "__doc__", None): | |
| # _bisect.bisect == _bisect.bisect_right in py31, but docs differ | |
| out(indent, p_name, " = ", seen[id(p_func)]) | |
| out(indent, "") | |
| return | |
| else: | |
| seen[id(p_func)] = p_name | |
| # real work | |
| if classname is None: | |
| classname = p_class and p_class.__name__ or None | |
| if p_class and hasattr(p_class, '__mro__'): | |
| sip_generated = [base_t for base_t in p_class.__mro__ if 'sip.simplewrapper' in str(base_t)] | |
| else: | |
| sip_generated = False | |
| deco = None | |
| deco_comment = "" | |
| mod_class_method_tuple = (p_modname, classname, p_name) | |
| ret_literal = None | |
| is_init = False | |
| # any decorators? | |
| action("redoing decos of func %r of class %r", p_name, p_class) | |
| if self.doing_builtins and p_modname == BUILTIN_MOD_NAME: | |
| deco = KNOWN_DECORATORS.get((classname, p_name), None) | |
| if deco: | |
| deco_comment = " # known case" | |
| elif p_class and p_name in p_class.__dict__: | |
| # detect native methods declared with METH_CLASS flag | |
| descriptor = p_class.__dict__[p_name] | |
| if p_name != "__new__" and type(descriptor).__name__.startswith('classmethod'): | |
| # 'classmethod_descriptor' in Python 2.x and 3.x, 'classmethod' in Jython | |
| deco = "classmethod" | |
| elif type(p_func).__name__.startswith('staticmethod'): | |
| deco = "staticmethod" | |
| if p_name == "__new__": | |
| deco = "staticmethod" | |
| deco_comment = " # known case of __new__" | |
| action("redoing innards of func %r of class %r", p_name, p_class) | |
| if deco and HAS_DECORATORS: | |
| out(indent, "@", deco, deco_comment) | |
| if inspect and inspect.isfunction(p_func): | |
| out(indent, "def ", p_name, restore_by_inspect(p_func), ": # reliably restored by inspect", ) | |
| out_doc_attr(out, p_func, indent + 1, p_class) | |
| elif self.is_predefined_builtin(*mod_class_method_tuple): | |
| spec, sig_note = restore_predefined_builtin(classname, p_name) | |
| out(indent, "def ", spec, ": # ", sig_note) | |
| out_doc_attr(out, p_func, indent + 1, p_class) | |
| elif sys.platform == 'cli' and is_clr_type(p_class): | |
| is_static, spec, sig_note = restore_clr(p_name, p_class) | |
| if is_static: | |
| out(indent, "@staticmethod") | |
| if not spec: return | |
| if sig_note: | |
| out(indent, "def ", spec, ": #", sig_note) | |
| else: | |
| out(indent, "def ", spec, ":") | |
| if not p_name in ['__gt__', '__ge__', '__lt__', '__le__', '__ne__', '__reduce_ex__', '__str__']: | |
| out_doc_attr(out, p_func, indent + 1, p_class) | |
| elif mod_class_method_tuple in PREDEFINED_MOD_CLASS_SIGS: | |
| sig, ret_literal = PREDEFINED_MOD_CLASS_SIGS[mod_class_method_tuple] | |
| if classname: | |
| ofwhat = "%s.%s.%s" % mod_class_method_tuple | |
| else: | |
| ofwhat = "%s.%s" % (p_modname, p_name) | |
| out(indent, "def ", p_name, sig, ": # known case of ", ofwhat) | |
| out_doc_attr(out, p_func, indent + 1, p_class) | |
| else: | |
| # __doc__ is our best source of arglist | |
| sig_note = "real signature unknown" | |
| spec = "" | |
| is_init = (p_name == "__init__" and p_class is not None) | |
| funcdoc = None | |
| if is_init and hasattr(p_class, "__doc__"): | |
| if hasattr(p_func, "__doc__"): | |
| funcdoc = p_func.__doc__ | |
| if funcdoc == object.__init__.__doc__: | |
| funcdoc = p_class.__doc__ | |
| elif hasattr(p_func, "__doc__"): | |
| funcdoc = p_func.__doc__ | |
| sig_restored = False | |
| action("parsing doc of func %r of class %r", p_name, p_class) | |
| if isinstance(funcdoc, STR_TYPES): | |
| (spec, ret_literal, more_notes) = self.parse_func_doc(funcdoc, p_name, p_name, classname, deco, | |
| sip_generated) | |
| if spec is None and p_name == '__init__' and classname: | |
| (spec, ret_literal, more_notes) = self.parse_func_doc(funcdoc, classname, p_name, classname, deco, | |
| sip_generated) | |
| sig_restored = spec is not None | |
| if more_notes: | |
| if sig_note: | |
| sig_note += "; " | |
| sig_note += more_notes | |
| if not sig_restored: | |
| # use an allow-all declaration | |
| decl = [] | |
| if p_class: | |
| first_param = propose_first_param(deco) | |
| if first_param: | |
| decl.append(first_param) | |
| decl.append("*args") | |
| decl.append("**kwargs") | |
| spec = p_name + "(" + ", ".join(decl) + ")" | |
| out(indent, "def ", spec, ": # ", sig_note) | |
| # to reduce size of stubs, don't output same docstring twice for class and its __init__ method | |
| if not is_init or funcdoc != p_class.__doc__: | |
| out_docstring(out, funcdoc, indent + 1) | |
| # body | |
| if ret_literal and not is_init: | |
| out(indent + 1, "return ", ret_literal) | |
| else: | |
| out(indent + 1, "pass") | |
| if deco and not HAS_DECORATORS: | |
| out(indent, p_name, " = ", deco, "(", p_name, ")", deco_comment) | |
| out(0, "") # empty line after each item | |
| def redo_class(self, out, p_class, p_name, indent, p_modname=None, seen=None, inspect_dir=False): | |
| """ | |
| Restores a class definition. | |
| @param out output function of a relevant buf | |
| @param p_class the class object | |
| @param p_name class name as known to owner | |
| @param indent indentation level | |
| @param p_modname name of module | |
| @param seen {class: name} map of classes already seen in the same namespace | |
| """ | |
| action("redoing class %r of module %r", p_name, p_modname) | |
| if seen is not None: | |
| if p_class in seen: | |
| out(indent, p_name, " = ", seen[p_class]) | |
| out(indent, "") | |
| return | |
| else: | |
| seen[p_class] = p_name | |
| bases = get_bases(p_class) | |
| base_def = "" | |
| skipped_bases = [] | |
| if bases: | |
| skip_qualifiers = [p_modname, BUILTIN_MOD_NAME, 'exceptions'] | |
| skip_qualifiers.extend(KNOWN_FAKE_REEXPORTERS.get(p_modname, ())) | |
| bases_list = [] # what we'll render in the class decl | |
| for base in bases: | |
| if [1 for (cls, mdl) in KNOWN_FAKE_BASES if cls == base and mdl != self.module]: | |
| # our base is a wrapper and our module is not its defining module | |
| skipped_bases.append(str(base)) | |
| continue | |
| # somehow import every base class | |
| base_name = base.__name__ | |
| qual_module_name = qualifier_of(base, skip_qualifiers) | |
| got_existing_import = False | |
| if qual_module_name: | |
| if qual_module_name in self.used_imports: | |
| import_list = self.used_imports[qual_module_name] | |
| if base in import_list: | |
| bases_list.append(base_name) # unqualified: already set to import | |
| got_existing_import = True | |
| if not got_existing_import: | |
| mangled_qualifier = "__" + qual_module_name.replace('.', '_') # foo.bar -> __foo_bar | |
| bases_list.append(mangled_qualifier + "." + base_name) | |
| self.hidden_imports[qual_module_name] = mangled_qualifier | |
| else: | |
| bases_list.append(base_name) | |
| base_def = "(" + ", ".join(bases_list) + ")" | |
| if self.split_modules: | |
| for base in bases_list: | |
| local_import = self.create_local_import(base) | |
| if local_import: | |
| out(indent, local_import) | |
| out(indent, "class ", p_name, base_def, ":", | |
| skipped_bases and " # skipped bases: " + ", ".join(skipped_bases) or "") | |
| out_doc_attr(out, p_class, indent + 1) | |
| # inner parts | |
| methods = {} | |
| properties = {} | |
| others = {} | |
| we_are_the_base_class = p_modname == BUILTIN_MOD_NAME and p_name == "object" | |
| field_source = {} | |
| try: | |
| if hasattr(p_class, "__dict__") and not inspect_dir: | |
| field_source = p_class.__dict__ | |
| field_keys = field_source.keys() # Jython 2.5.1 _codecs fail here | |
| else: | |
| field_keys = dir(p_class) # this includes unwanted inherited methods, but no dict + inheritance is rare | |
| except: | |
| field_keys = () | |
| for item_name in field_keys: | |
| item_qname = p_modname + '.' + p_name + '.' + item_name | |
| if item_name in ("__doc__", "__module__"): | |
| if we_are_the_base_class: | |
| item = "" # must be declared in base types | |
| else: | |
| continue # in all other cases must be skipped | |
| elif keyword.iskeyword(item_name): # for example, PyQt4 contains definitions of methods named 'exec' | |
| continue | |
| elif item_qname in CLASS_ATTR_BLACKLIST: | |
| note('skipping blacklisted attribute ' + item_qname) | |
| item = field_source.get(item_name) | |
| else: | |
| try: | |
| item = getattr(p_class, item_name) # let getters do the magic | |
| except AttributeError: | |
| item = field_source.get(item_name) # have it raw | |
| if item is None: | |
| continue | |
| except Exception: | |
| continue | |
| if is_callable(item) and not isinstance(item, type): | |
| methods[item_name] = item | |
| elif is_property(item): | |
| properties[item_name] = item | |
| else: | |
| others[item_name] = item | |
| # | |
| if we_are_the_base_class: | |
| others["__dict__"] = {} # force-feed it, for __dict__ does not contain a reference to itself :) | |
| # add fake __init__s to have the right sig | |
| if p_class in FAKE_BUILTIN_INITS: | |
| methods["__init__"] = self.fake_builtin_init | |
| note("Faking init of %s", p_name) | |
| elif '__init__' not in methods: | |
| init_method = getattr(p_class, '__init__', None) | |
| if init_method: | |
| methods['__init__'] = init_method | |
| # | |
| seen_funcs = {} | |
| for item_name in sorted_no_case(methods.keys()): | |
| item = methods[item_name] | |
| try: | |
| self.redo_function(out, item, item_name, indent + 1, p_class, p_modname, classname=p_name, seen=seen_funcs) | |
| except: | |
| handle_error_func(item_name, out) | |
| # | |
| known_props = KNOWN_PROPS.get(p_modname, {}) | |
| a_setter = "lambda self, v: None" | |
| a_deleter = "lambda self: None" | |
| for item_name in sorted_no_case(properties.keys()): | |
| item = properties[item_name] | |
| prop_docstring = getattr(item, '__doc__', None) | |
| prop_key = (p_name, item_name) | |
| if prop_key in known_props: | |
| prop_descr = known_props.get(prop_key, None) | |
| if prop_descr is None: | |
| continue # explicitly omitted | |
| acc_line, getter_and_type = prop_descr | |
| if getter_and_type: | |
| getter, prop_type = getter_and_type | |
| else: | |
| getter, prop_type = None, None | |
| out(indent + 1, item_name, | |
| " = property(", format_accessors(acc_line, getter, a_setter, a_deleter), ")" | |
| ) | |
| if prop_type: | |
| if prop_docstring: | |
| out(indent + 1, '"""', prop_docstring) | |
| out(0, "") | |
| out(indent + 1, ':type: ', prop_type) | |
| out(indent + 1, '"""') | |
| else: | |
| out(indent + 1, '""":type: ', prop_type, '"""') | |
| out(0, "") | |
| else: | |
| out(indent + 1, item_name, " = property(lambda self: object(), lambda self, v: None, lambda self: None) # default") | |
| if prop_docstring: | |
| out(indent + 1, '"""', prop_docstring, '"""') | |
| out(0, "") | |
| if properties: | |
| out(0, "") # empty line after the block | |
| # | |
| for item_name in sorted_no_case(others.keys()): | |
| item = others[item_name] | |
| self.fmt_value(out, item, indent + 1, prefix=item_name + " = ") | |
| if p_name == "object": | |
| out(indent + 1, "__module__ = ''") | |
| if others: | |
| out(0, "") # empty line after the block | |
| # | |
| if not methods and not properties and not others: | |
| out(indent + 1, "pass") | |
| def redo_simple_header(self, p_name): | |
| """Puts boilerplate code on the top""" | |
| out = self.header_buf.out # 1st class methods rule :) | |
| out(0, "# encoding: %s" % OUT_ENCODING) # line 1 | |
| # NOTE: maybe encoding should be selectable | |
| if hasattr(self.module, "__name__"): | |
| self_name = self.module.__name__ | |
| if self_name != p_name: | |
| mod_name = " calls itself " + self_name | |
| else: | |
| mod_name = "" | |
| else: | |
| mod_name = " does not know its name" | |
| out(0, "# module ", p_name, mod_name) # line 2 | |
| origin_type = OriginType.FILE | |
| import generator3.core | |
| if generator3.core.is_pregeneration_mode(): | |
| origin = origin_type = OriginType.PREGENERATED | |
| elif self.mod_filename: | |
| origin = self.mod_filename | |
| elif p_name in sys.builtin_module_names: | |
| origin = origin_type = OriginType.BUILTIN | |
| else: | |
| try: | |
| origin = getattr(self.module, "__file__") | |
| except AttributeError: | |
| origin = origin_type = OriginType.BUILTIN | |
| if self.test_mode and origin_type == OriginType.FILE: | |
| origin = get_relative_path_by_qname(origin, self.qname) | |
| out(0, "# from %s" % origin) # line 3 | |
| out(0, "# by generator %s" % self.gen_version) # line 4 | |
| if p_name == BUILTIN_MOD_NAME and version[0] == 2 and version[1] >= 6: | |
| out(0, "from __future__ import print_function") | |
| out_doc_attr(out, self.module, 0) | |
| def redo_imports(self): | |
| module_type = type(sys) | |
| for item_name in self.module.__dict__.keys(): | |
| try: | |
| item = self.module.__dict__[item_name] | |
| except: | |
| continue | |
| if type(item) is module_type: # not isinstance, py2.7 + PyQt4.QtCore on windows have a bug here | |
| self.imported_modules[item_name] = item | |
| self.add_import_header_if_needed() | |
| ref_notice = getattr(item, "__file__", str(item)) if not self.test_mode else '<ref>' | |
| if hasattr(item, "__name__"): | |
| self.imports_buf.out(0, "import ", item.__name__, " as ", item_name, " # ", ref_notice) | |
| else: | |
| self.imports_buf.out(0, item_name, " = None # ??? name unknown; ", ref_notice) | |
| def add_import_header_if_needed(self): | |
| if self.imports_buf.isEmpty(): | |
| self.imports_buf.out(0, "") | |
| self.imports_buf.out(0, "# imports") | |
| def redo(self, p_name, inspect_dir): | |
| """ | |
| Restores module declarations. | |
| Intended for built-in modules and thus does not handle import statements. | |
| @param p_name name of module | |
| """ | |
| action("redoing header of module %r %r", p_name, str(self.module)) | |
| if "pyqt4" in p_name.lower(): # qt4 specific patch | |
| self._initializeQApp4() | |
| elif "pyqt5" in p_name.lower(): # qt5 specific patch | |
| self._initializeQApp5() | |
| self.redo_simple_header(p_name) | |
| # find whatever other self.imported_modules the module knows; effectively these are imports | |
| action("redoing imports of module %r %r", p_name, str(self.module)) | |
| try: | |
| self.redo_imports() | |
| except: | |
| pass | |
| action("redoing innards of module %r %r", p_name, str(self.module)) | |
| module_type = type(sys) | |
| # group what we have into buckets | |
| vars_simple = {} | |
| vars_complex = {} | |
| funcs = {} | |
| classes = {} | |
| module_dict = self.module.__dict__ | |
| if inspect_dir: | |
| module_dict = dir(self.module) | |
| for item_name in module_dict: | |
| note("looking at %s", item_name) | |
| if item_name in ( | |
| "__dict__", "__doc__", "__module__", "__file__", "__name__", "__builtins__", "__package__"): | |
| continue # handled otherwise | |
| if self.test_mode and item_name in ('__loader__', '__spec__', '__cached__'): | |
| continue | |
| try: | |
| item = getattr(self.module, item_name) # let getters do the magic | |
| except AttributeError: | |
| if not item_name in self.module.__dict__: continue | |
| item = self.module.__dict__[item_name] # have it raw | |
| # check if it has percolated from an imported module | |
| except NotImplementedError: | |
| if not item_name in self.module.__dict__: continue | |
| item = self.module.__dict__[item_name] # have it raw | |
| # unless we're adamantly positive that the name was imported, we assume it is defined here | |
| mod_name = None # module from which p_name might have been imported | |
| # IronPython has non-trivial reexports in System module, but not in others: | |
| skip_modname = sys.platform == "cli" and p_name != "System" | |
| surely_not_imported_mods = KNOWN_FAKE_REEXPORTERS.get(p_name, ()) | |
| ## can't figure weirdness in some modules, assume no reexports: | |
| #skip_modname = skip_modname or p_name in self.KNOWN_FAKE_REEXPORTERS | |
| if not skip_modname: | |
| try: | |
| mod_name = getattr(item, '__module__', None) | |
| except: | |
| pass | |
| # we assume that module foo.bar never imports foo; foo may import foo.bar. (see pygame and pygame.rect) | |
| maybe_import_mod_name = mod_name if isinstance(mod_name, type(p_name)) else '' | |
| import_is_from_top = len(p_name) > len(maybe_import_mod_name) and p_name.startswith(maybe_import_mod_name) | |
| note("mod_name = %s, prospective = %s, from top = %s", mod_name, maybe_import_mod_name, import_is_from_top) | |
| want_to_import = False | |
| if (mod_name | |
| and mod_name != BUILTIN_MOD_NAME | |
| and mod_name != p_name | |
| and mod_name not in surely_not_imported_mods | |
| and not import_is_from_top | |
| ): | |
| # import looks valid, but maybe it's a .py file? we're certain not to import from .py | |
| # e.g. this rules out _collections import collections and builtins import site. | |
| try: | |
| imported = __import__(mod_name) # ok to repeat, Python caches for us | |
| if imported: | |
| qualifiers = mod_name.split(".")[1:] | |
| for qual in qualifiers: | |
| imported = getattr(imported, qual, None) | |
| if not imported: | |
| break | |
| imported_path = (getattr(imported, '__file__', False) or "").lower() | |
| want_to_import = not (imported_path.endswith('.py') or imported_path.endswith('.pyc')) | |
| imported_name = getattr(imported, "__name__", None) | |
| if imported_name == p_name: | |
| want_to_import = False | |
| note("path of %r is %r, want? %s", mod_name, imported_path, want_to_import) | |
| except ImportError: | |
| want_to_import = False | |
| # NOTE: if we fail to import, we define 'imported' names here lest we lose them at all | |
| if want_to_import: | |
| import_list = self.used_imports[mod_name] | |
| if item_name not in import_list: | |
| import_list.append(item_name) | |
| if not want_to_import: | |
| if isinstance(item, type) or type(item).__name__ == 'classobj': | |
| classes[item_name] = item | |
| elif is_callable(item): # some classes are callable, check them before functions | |
| funcs[item_name] = item | |
| elif isinstance(item, module_type): | |
| continue # self.imported_modules handled above already | |
| else: | |
| if isinstance(item, SIMPLEST_TYPES): | |
| vars_simple[item_name] = item | |
| else: | |
| vars_complex[item_name] = item | |
| # sort and output every bucket | |
| action("outputting innards of module %r %r", p_name, str(self.module)) | |
| # | |
| omitted_names = OMIT_NAME_IN_MODULE.get(p_name, []) | |
| if vars_simple: | |
| out = self.functions_buf.out | |
| prefix = "" # try to group variables by common prefix | |
| PREFIX_LEN = 2 # default prefix length if we can't guess better | |
| out(0, "# Variables with simple values") | |
| for item_name in sorted_no_case(vars_simple.keys()): | |
| if item_name in omitted_names: | |
| out(0, "# definition of " + item_name + " omitted") | |
| continue | |
| item = vars_simple[item_name] | |
| # track the prefix | |
| if len(item_name) >= PREFIX_LEN: | |
| prefix_pos = string.rfind(item_name, "_") # most prefixes end in an underscore | |
| if prefix_pos < 1: | |
| prefix_pos = PREFIX_LEN | |
| beg = item_name[0:prefix_pos] | |
| if prefix != beg: | |
| out(0, "") # space out from other prefix | |
| prefix = beg | |
| else: | |
| prefix = "" | |
| # output | |
| replacement = REPLACE_MODULE_VALUES.get((p_name, item_name), None) | |
| if replacement is not None: | |
| out(0, item_name, " = ", replacement, " # real value of type ", str(type(item)), " replaced") | |
| elif is_skipped_in_module(p_name, item_name): | |
| t_item = type(item) | |
| out(0, item_name, " = ", self.invent_initializer(t_item), " # real value of type ", str(t_item), | |
| " skipped") | |
| else: | |
| self.fmt_value(out, item, 0, prefix=item_name + " = ") | |
| self._defined[item_name] = True | |
| out(0, "") # empty line after vars | |
| # | |
| if funcs: | |
| out = self.functions_buf.out | |
| out(0, "# functions") | |
| out(0, "") | |
| seen_funcs = {} | |
| for item_name in sorted_no_case(funcs.keys()): | |
| if item_name in omitted_names: | |
| out(0, "# definition of ", item_name, " omitted") | |
| continue | |
| item = funcs[item_name] | |
| try: | |
| self.redo_function(out, item, item_name, 0, p_modname=p_name, seen=seen_funcs) | |
| except: | |
| handle_error_func(item_name, out) | |
| else: | |
| self.functions_buf.out(0, "# no functions") | |
| # | |
| if classes: | |
| self.classes_buf.out(0, "# classes") | |
| self.classes_buf.out(0, "") | |
| seen_classes = {} | |
| # sort classes so that inheritance order is preserved | |
| cls_list = [] # items are (class_name, mro_tuple) | |
| for cls_name in sorted_no_case(classes.keys()): | |
| cls = classes[cls_name] | |
| ins_index = len(cls_list) | |
| for i in range(ins_index): | |
| maybe_child_bases = cls_list[i][1] | |
| if cls in maybe_child_bases: | |
| ins_index = i # we could not go farther than current ins_index | |
| break # ...and need not go fartehr than first known child | |
| cls_list.insert(ins_index, (cls_name, get_mro(cls))) | |
| self.split_modules = self.mod_filename and len(cls_list) >= 30 | |
| for item_name in [cls_item[0] for cls_item in cls_list]: | |
| buf = ClassBuf(item_name, self) | |
| self.classes_buffs.append(buf) | |
| out = buf.out | |
| if item_name in omitted_names: | |
| out(0, "# definition of ", item_name, " omitted") | |
| continue | |
| item = classes[item_name] | |
| self.redo_class(out, item, item_name, 0, p_modname=p_name, seen=seen_classes, inspect_dir=inspect_dir) | |
| self._defined[item_name] = True | |
| out(0, "") # empty line after each item | |
| if self.doing_builtins and p_name == BUILTIN_MOD_NAME and version[0] < 3: | |
| # classobj still supported | |
| txt = classobj_txt | |
| self.classes_buf.out(0, txt) | |
| if self.doing_builtins and p_name == BUILTIN_MOD_NAME: | |
| txt = create_generator() | |
| self.classes_buf.out(0, txt) | |
| txt = create_async_generator() | |
| self.classes_buf.out(0, txt) | |
| txt = create_function() | |
| self.classes_buf.out(0, txt) | |
| txt = create_method() | |
| self.classes_buf.out(0, txt) | |
| txt = create_coroutine() | |
| self.classes_buf.out(0, txt) | |
| # Fake <type 'namedtuple'> | |
| if version[0] >= 3 or (version[0] == 2 and version[1] >= 6): | |
| namedtuple_text = create_named_tuple() | |
| self.classes_buf.out(0, namedtuple_text) | |
| else: | |
| self.classes_buf.out(0, "# no classes") | |
| # | |
| if vars_complex: | |
| out = self.footer_buf.out | |
| out(0, "# variables with complex values") | |
| out(0, "") | |
| for item_name in sorted_no_case(vars_complex.keys()): | |
| if item_name in omitted_names: | |
| out(0, "# definition of " + item_name + " omitted") | |
| continue | |
| item = vars_complex[item_name] | |
| if str(type(item)) == "<type 'namespace#'>": | |
| continue # this is an IronPython submodule, we mustn't generate a reference for it in the base module | |
| replacement = REPLACE_MODULE_VALUES.get((p_name, item_name), None) | |
| if replacement is not None: | |
| out(0, item_name + " = " + replacement + " # real value of type " + str(type(item)) + " replaced") | |
| elif is_skipped_in_module(p_name, item_name): | |
| t_item = type(item) | |
| out(0, item_name + " = " + self.invent_initializer(t_item) + " # real value of type " + str( | |
| t_item) + " skipped") | |
| else: | |
| self.fmt_value(out, item, 0, prefix=item_name + " = ", as_name=item_name) | |
| self._defined[item_name] = True | |
| out(0, "") # empty line after each item | |
| values_to_add = ADD_VALUE_IN_MODULE.get(p_name, None) | |
| if values_to_add: | |
| self.footer_buf.out(0, "# intermittent names") | |
| for value in values_to_add: | |
| self.footer_buf.out(0, value) | |
| # imports: last, because previous parts could alter used_imports or hidden_imports | |
| self.output_import_froms() | |
| if self.imports_buf.isEmpty(): | |
| self.imports_buf.out(0, "# no imports") | |
| self.imports_buf.out(0, "") # empty line after imports | |
| def output_import_froms(self): | |
| """Mention all imported names known within the module, wrapping as per PEP.""" | |
| out = self.imports_buf.out | |
| if self.used_imports: | |
| self.add_import_header_if_needed() | |
| for mod_name in sorted_no_case(self.used_imports.keys()): | |
| import_names = self.used_imports[mod_name] | |
| if import_names: | |
| self._defined[mod_name] = True | |
| right_pos = 0 # tracks width of list to fold it at right margin | |
| import_heading = "from % s import (" % mod_name | |
| right_pos += len(import_heading) | |
| names_pack = [import_heading] | |
| indent_level = 0 | |
| import_names = list(import_names) | |
| import_names.sort() | |
| for n in import_names: | |
| self._defined[n] = True | |
| len_n = len(n) | |
| if right_pos + len_n >= 78: | |
| out(indent_level, *names_pack) | |
| names_pack = [n, ", "] | |
| if indent_level == 0: | |
| indent_level = 1 # all but first line is indented | |
| right_pos = self.indent_size + len_n + 2 | |
| else: | |
| names_pack.append(n) | |
| names_pack.append(", ") | |
| right_pos += (len_n + 2) | |
| # last line is... | |
| if indent_level == 0: # one line | |
| names_pack[0] = names_pack[0][:-1] # cut off lpar | |
| names_pack[-1] = "" # cut last comma | |
| else: # last line of multiline | |
| names_pack[-1] = ")" # last comma -> rpar | |
| out(indent_level, *names_pack) | |
| out(0, "") # empty line after group | |
| if self.hidden_imports: | |
| self.add_import_header_if_needed() | |
| for mod_name in sorted_no_case(self.hidden_imports.keys()): | |
| out(0, 'import ', mod_name, ' as ', self.hidden_imports[mod_name]) | |
| out(0, "") # empty line after group | |