| # epydoc -- Source code parsing |
| # |
| # Copyright (C) 2005 Edward Loper |
| # Author: Edward Loper <edloper@loper.org> |
| # URL: <http://epydoc.sf.net> |
| # |
| # $Id: docparser.py 1673 2008-01-29 05:42:58Z edloper $ |
| |
| """ |
| Extract API documentation about python objects by parsing their source |
| code. |
| |
| The function L{parse_docs()}, which provides the main interface |
| of this module, reads and parses the Python source code for a |
| module, and uses it to create an L{APIDoc} object containing |
| the API documentation for the variables and values defined in |
| that modules. |
| |
| Currently, C{parse_docs()} extracts documentation from the following |
| source code constructions: |
| |
| - module docstring |
| - import statements |
| - class definition blocks |
| - function definition blocks |
| - assignment statements |
| - simple assignment statements |
| - assignment statements with multiple C{'='}s |
| - assignment statements with unpacked left-hand sides |
| - assignment statements that wrap a function in classmethod |
| or staticmethod. |
| - assignment to special variables __path__, __all__, and |
| __docformat__. |
| - delete statements |
| |
| C{parse_docs()} does not yet support the following source code |
| constructions: |
| |
| - assignment statements that create properties |
| |
| By default, C{parse_docs()} will expore the contents of top-level |
| C{try} and C{if} blocks. If desired, C{parse_docs()} can also |
| be configured to explore the contents of C{while} and C{for} blocks. |
| (See the configuration constants, below.) |
| |
| @todo: Make it possible to extend the functionality of C{parse_docs()}, |
| by replacing process_line with a dispatch table that can be |
| customized (similarly to C{docintrospector.register_introspector()}). |
| """ |
| __docformat__ = 'epytext en' |
| |
| ###################################################################### |
| ## Imports |
| ###################################################################### |
| |
| # Python source code parsing: |
| import token, tokenize |
| # Finding modules: |
| import imp |
| # File services: |
| import os, os.path, sys |
| # Unicode: |
| import codecs |
| # API documentation encoding: |
| from epydoc.apidoc import * |
| # For looking up the docs of builtins: |
| import __builtin__, exceptions |
| import epydoc.docintrospecter |
| # Misc utility functions: |
| from epydoc.util import * |
| # Backwards compatibility |
| from epydoc.compat import * |
| |
| ###################################################################### |
| ## Doc Parser |
| ###################################################################### |
| |
| class ParseError(Exception): |
| """ |
| An exception that is used to signify that C{docparser} encountered |
| syntactically invalid Python code while processing a Python source |
| file. |
| """ |
| |
| _moduledoc_cache = {} |
| """A cache of C{ModuleDoc}s that we've already created. |
| C{_moduledoc_cache} is a dictionary mapping from filenames to |
| C{ValueDoc} objects. |
| @type: C{dict}""" |
| |
| #//////////////////////////////////////////////////////////// |
| # Configuration Constants |
| #//////////////////////////////////////////////////////////// |
| |
| #{ Configuration Constants: Control Flow |
| PARSE_TRY_BLOCKS = True |
| """Should the contents of C{try} blocks be examined?""" |
| PARSE_EXCEPT_BLOCKS = True |
| """Should the contents of C{except} blocks be examined?""" |
| PARSE_FINALLY_BLOCKS = True |
| """Should the contents of C{finally} blocks be examined?""" |
| PARSE_IF_BLOCKS = True |
| """Should the contents of C{if} blocks be examined?""" |
| PARSE_ELSE_BLOCKS = True |
| """Should the contents of C{else} and C{elif} blocks be examined?""" |
| PARSE_WHILE_BLOCKS = False |
| """Should the contents of C{while} blocks be examined?""" |
| PARSE_FOR_BLOCKS = False |
| """Should the contents of C{for} blocks be examined?""" |
| |
| #{ Configuration Constants: Imports |
| IMPORT_HANDLING = 'link' |
| """What should C{docparser} do when it encounters an import |
| statement? |
| - C{'link'}: Create variabledoc objects with imported_from pointers |
| to the source object. |
| - C{'parse'}: Parse the imported file, to find the actual |
| documentation for the imported object. (This will fall back |
| to the 'link' behavior if the imported file can't be parsed, |
| e.g., if it's a builtin.) |
| """ |
| |
| IMPORT_STAR_HANDLING = 'parse' |
| """When C{docparser} encounters a C{'from M{m} import *'} |
| statement, and is unable to parse C{M{m}} (either because |
| L{IMPORT_HANDLING}=C{'link'}, or because parsing failed), how |
| should it determine the list of identifiers expored by C{M{m}}? |
| - C{'ignore'}: ignore the import statement, and don't create |
| any new variables. |
| - C{'parse'}: parse it to find a list of the identifiers that it |
| exports. (This will fall back to the 'ignore' behavior if the |
| imported file can't be parsed, e.g., if it's a builtin.) |
| - C{'introspect'}: import the module and introspect it (using C{dir}) |
| to find a list of the identifiers that it exports. (This will |
| fall back to the 'ignore' behavior if the imported file can't |
| be parsed, e.g., if it's a builtin.) |
| """ |
| |
| DEFAULT_DECORATOR_BEHAVIOR = 'transparent' |
| """When C{DocParse} encounters an unknown decorator, what should |
| it do to the documentation of the decorated function? |
| - C{'transparent'}: leave the function's documentation as-is. |
| - C{'opaque'}: replace the function's documentation with an |
| empty C{ValueDoc} object, reflecting the fact that we have no |
| knowledge about what value the decorator returns. |
| """ |
| |
| BASE_HANDLING = 'parse'#'link' |
| """What should C{docparser} do when it encounters a base class that |
| was imported from another module? |
| - C{'link'}: Create a valuedoc with a C{proxy_for} pointer to the |
| base class. |
| - C{'parse'}: Parse the file containing the base class, to find |
| the actual documentation for it. (This will fall back to the |
| 'link' behavior if the imported file can't be parsed, e.g., if |
| it's a builtin.) |
| """ |
| |
| #{ Configuration Constants: Comment docstrings |
| COMMENT_DOCSTRING_MARKER = '#:' |
| """The prefix used to mark comments that contain attribute |
| docstrings for variables.""" |
| |
| #{ Configuration Constants: Grouping |
| START_GROUP_MARKER = '#{' |
| """The prefix used to mark a comment that starts a group. This marker |
| should be followed (on the same line) by the name of the group. |
| Following a start-group comment, all variables defined at the same |
| indentation level will be assigned to this group name, until the |
| parser reaches the end of the file, a matching end-group comment, or |
| another start-group comment at the same indentation level. |
| """ |
| |
| END_GROUP_MARKER = '#}' |
| """The prefix used to mark a comment that ends a group. See |
| L{START_GROUP_MARKER}.""" |
| |
| #///////////////////////////////////////////////////////////////// |
| #{ Module parser |
| #///////////////////////////////////////////////////////////////// |
| |
| def parse_docs(filename=None, name=None, context=None, is_script=False): |
| """ |
| Generate the API documentation for a specified object by |
| parsing Python source files, and return it as a L{ValueDoc}. |
| The object to generate documentation for may be specified |
| using the C{filename} parameter I{or} the C{name} parameter. |
| (It is an error to specify both a filename and a name; or to |
| specify neither a filename nor a name). |
| |
| @param filename: The name of the file that contains the python |
| source code for a package, module, or script. If |
| C{filename} is specified, then C{parse} will return a |
| C{ModuleDoc} describing its contents. |
| @param name: The fully-qualified python dotted name of any |
| value (including packages, modules, classes, and |
| functions). C{parse_docs()} will automatically figure out |
| which module(s) it needs to parse in order to find the |
| documentation for the specified object. |
| @param context: The API documentation for the package that |
| contains C{filename}. If no context is given, then |
| C{filename} is assumed to contain a top-level module or |
| package. It is an error to specify a C{context} if the |
| C{name} argument is used. |
| @rtype: L{ValueDoc} |
| """ |
| # Always introspect __builtins__ & exceptions (e.g., in case |
| # they're used as base classes.) |
| epydoc.docintrospecter.introspect_docs(__builtin__) |
| epydoc.docintrospecter.introspect_docs(exceptions) |
| |
| # If our input is a python object name, then delegate to |
| # _find(). |
| if filename is None and name is not None: |
| if context: |
| raise ValueError("context should only be specified together " |
| "with filename, not with name.") |
| name = DottedName(name) |
| val_doc = _find(name) |
| if val_doc.canonical_name is UNKNOWN: |
| val_doc.canonical_name = name |
| return val_doc |
| |
| # If our input is a filename, then create a ModuleDoc for it, |
| # and use process_file() to populate its attributes. |
| elif filename is not None and name is None: |
| # Use a python source version, if possible. |
| if not is_script: |
| try: filename = py_src_filename(filename) |
| except ValueError, e: raise ImportError('%s' % e) |
| |
| # Check the cache, first. |
| if filename in _moduledoc_cache: |
| return _moduledoc_cache[filename] |
| |
| log.info("Parsing %s" % filename) |
| |
| # If the context wasn't provided, then check if the file is in |
| # a package directory. If so, then update basedir & name to |
| # contain the topmost package's directory and the fully |
| # qualified name for this file. (This update assume the |
| # default value of __path__ for the parent packages; if the |
| # parent packages override their __path__s, then this can |
| # cause us not to find the value.) |
| if context is None and not is_script: |
| basedir = os.path.split(filename)[0] |
| name = os.path.splitext(os.path.split(filename)[1])[0] |
| if name == '__init__': |
| basedir, name = os.path.split(basedir) |
| context = _parse_package(basedir) |
| |
| # Figure out the canonical name of the module we're parsing. |
| if not is_script: |
| module_name, is_pkg = _get_module_name(filename, context) |
| else: |
| module_name = DottedName(munge_script_name(filename)) |
| is_pkg = False |
| |
| # Create a new ModuleDoc for the module, & add it to the cache. |
| module_doc = ModuleDoc(canonical_name=module_name, variables={}, |
| sort_spec=[], imports=[], |
| filename=filename, package=context, |
| is_package=is_pkg, submodules=[], |
| docs_extracted_by='parser') |
| module_doc.defining_module = module_doc |
| _moduledoc_cache[filename] = module_doc |
| |
| # Set the module's __path__ to its default value. |
| if is_pkg: |
| module_doc.path = [os.path.split(module_doc.filename)[0]] |
| |
| # Add this module to the parent package's list of submodules. |
| if context is not None: |
| context.submodules.append(module_doc) |
| |
| # Tokenize & process the contents of the module's source file. |
| try: |
| process_file(module_doc) |
| except tokenize.TokenError, e: |
| msg, (srow, scol) = e.args |
| raise ParseError('Error during parsing: %s ' |
| '(%s, line %d, char %d)' % |
| (msg, module_doc.filename, srow, scol)) |
| except IndentationError, e: |
| raise ParseError('Error during parsing: %s (%s)' % |
| (e, module_doc.filename)) |
| |
| # Handle any special variables (__path__, __docformat__, etc.) |
| handle_special_module_vars(module_doc) |
| |
| # Return the completed ModuleDoc |
| return module_doc |
| else: |
| raise ValueError("Expected exactly one of the following " |
| "arguments: name, filename") |
| |
| def _parse_package(package_dir): |
| """ |
| If the given directory is a package directory, then parse its |
| __init__.py file (and the __init__.py files of all ancestor |
| packages); and return its C{ModuleDoc}. |
| """ |
| if not is_package_dir(package_dir): |
| return None |
| parent_dir = os.path.split(package_dir)[0] |
| parent_doc = _parse_package(parent_dir) |
| package_file = os.path.join(package_dir, '__init__') |
| return parse_docs(filename=package_file, context=parent_doc) |
| |
| # Special vars: |
| # C{__docformat__}, C{__all__}, and C{__path__}. |
| def handle_special_module_vars(module_doc): |
| # If __docformat__ is defined, parse its value. |
| toktree = _module_var_toktree(module_doc, '__docformat__') |
| if toktree is not None: |
| try: module_doc.docformat = parse_string(toktree) |
| except: pass |
| del module_doc.variables['__docformat__'] |
| |
| # If __all__ is defined, parse its value. |
| toktree = _module_var_toktree(module_doc, '__all__') |
| if toktree is not None: |
| try: |
| public_names = set(parse_string_list(toktree)) |
| for name, var_doc in module_doc.variables.items(): |
| if name in public_names: |
| var_doc.is_public = True |
| if not isinstance(var_doc, ModuleDoc): |
| var_doc.is_imported = False |
| else: |
| var_doc.is_public = False |
| except ParseError: |
| # If we couldn't parse the list, give precedence to introspection. |
| for name, var_doc in module_doc.variables.items(): |
| if not isinstance(var_doc, ModuleDoc): |
| var_doc.is_imported = UNKNOWN |
| del module_doc.variables['__all__'] |
| |
| # If __path__ is defined, then extract its value (pkgs only) |
| if module_doc.is_package: |
| toktree = _module_var_toktree(module_doc, '__path__') |
| if toktree is not None: |
| try: |
| module_doc.path = parse_string_list(toktree) |
| except ParseError: |
| pass # [xx] |
| del module_doc.variables['__path__'] |
| |
| def _module_var_toktree(module_doc, name): |
| var_doc = module_doc.variables.get(name) |
| if (var_doc is None or var_doc.value in (None, UNKNOWN) or |
| var_doc.value.toktree is UNKNOWN): |
| return None |
| else: |
| return var_doc.value.toktree |
| |
| #//////////////////////////////////////////////////////////// |
| #{ Module Lookup |
| #//////////////////////////////////////////////////////////// |
| |
| def _find(name, package_doc=None): |
| """ |
| Return the API documentaiton for the object whose name is |
| C{name}. C{package_doc}, if specified, is the API |
| documentation for the package containing the named object. |
| """ |
| # If we're inside a package, then find the package's path. |
| if package_doc is None: |
| path = None |
| elif package_doc.path is not UNKNOWN: |
| path = package_doc.path |
| else: |
| path = [os.path.split(package_doc.filename)[0]] |
| |
| # The leftmost identifier in `name` should be a module or |
| # package on the given path; find it and parse it. |
| filename = _get_filename(name[0], path) |
| module_doc = parse_docs(filename, context=package_doc) |
| |
| # If the name just has one identifier, then the module we just |
| # parsed is the object we're looking for; return it. |
| if len(name) == 1: return module_doc |
| |
| # Otherwise, we're looking for something inside the module. |
| # First, check to see if it's in a variable (but ignore |
| # variables that just contain imported submodules). |
| if not _is_submodule_import_var(module_doc, name[1]): |
| try: return _find_in_namespace(name[1:], module_doc) |
| except ImportError: pass |
| |
| # If not, then check to see if it's in a subpackage. |
| if module_doc.is_package: |
| return _find(name[1:], module_doc) |
| |
| # If it's not in a variable or a subpackage, then we can't |
| # find it. |
| raise ImportError('Could not find value') |
| |
| def _is_submodule_import_var(module_doc, var_name): |
| """ |
| Return true if C{var_name} is the name of a variable in |
| C{module_doc} that just contains an C{imported_from} link to a |
| submodule of the same name. (I.e., is a variable created when |
| a package imports one of its own submodules.) |
| """ |
| var_doc = module_doc.variables.get(var_name) |
| full_var_name = DottedName(module_doc.canonical_name, var_name) |
| return (var_doc is not None and |
| var_doc.imported_from == full_var_name) |
| |
| def _find_in_namespace(name, namespace_doc): |
| if name[0] not in namespace_doc.variables: |
| raise ImportError('Could not find value') |
| |
| # Look up the variable in the namespace. |
| var_doc = namespace_doc.variables[name[0]] |
| if var_doc.value is UNKNOWN: |
| raise ImportError('Could not find value') |
| val_doc = var_doc.value |
| |
| # If the variable's value was imported, then follow its |
| # alias link. |
| if var_doc.imported_from not in (None, UNKNOWN): |
| return _find(var_doc.imported_from+name[1:]) |
| |
| # Otherwise, if the name has one identifier, then this is the |
| # value we're looking for; return it. |
| elif len(name) == 1: |
| return val_doc |
| |
| # Otherwise, if this value is a namespace, look inside it. |
| elif isinstance(val_doc, NamespaceDoc): |
| return _find_in_namespace(name[1:], val_doc) |
| |
| # Otherwise, we ran into a dead end. |
| else: |
| raise ImportError('Could not find value') |
| |
| def _get_filename(identifier, path=None): |
| if path is UNKNOWN: path = None |
| try: |
| fp, filename, (s,m,typ) = imp.find_module(identifier, path) |
| if fp is not None: fp.close() |
| except ImportError: |
| raise ImportError, 'No Python source file found.' |
| |
| if typ == imp.PY_SOURCE: |
| return filename |
| elif typ == imp.PY_COMPILED: |
| # See if we can find a corresponding non-compiled version. |
| filename = re.sub('.py\w$', '.py', filename) |
| if not os.path.exists(filename): |
| raise ImportError, 'No Python source file found.' |
| return filename |
| elif typ == imp.PKG_DIRECTORY: |
| filename = os.path.join(filename, '__init__.py') |
| if not os.path.exists(filename): |
| filename = os.path.join(filename, '__init__.pyw') |
| if not os.path.exists(filename): |
| raise ImportError, 'No package file found.' |
| return filename |
| elif typ == imp.C_BUILTIN: |
| raise ImportError, 'No Python source file for builtin modules.' |
| elif typ == imp.C_EXTENSION: |
| raise ImportError, 'No Python source file for c extensions.' |
| else: |
| raise ImportError, 'No Python source file found.' |
| |
| #///////////////////////////////////////////////////////////////// |
| #{ File tokenization loop |
| #///////////////////////////////////////////////////////////////// |
| |
| def process_file(module_doc): |
| """ |
| Read the given C{ModuleDoc}'s file, and add variables |
| corresponding to any objects defined in that file. In |
| particular, read and tokenize C{module_doc.filename}, and |
| process each logical line using L{process_line()}. |
| """ |
| # Keep track of the current line number: |
| lineno = None |
| |
| # Use this list to collect the tokens on a single logical line: |
| line_toks = [] |
| |
| # This list contains one APIDoc for each indentation level. |
| # The first element is the APIDoc for the module, and each |
| # subsequent element is the APIDoc for the object at that |
| # indentation level. The final element of the list is the |
| # C{APIDoc} for the entity that we're currently processing. |
| parent_docs = [module_doc] |
| |
| # The APIDoc for the object that was defined by the previous |
| # line, if any; or None otherwise. This is used to update |
| # parent_docs when we encounter an indent; and to decide what |
| # object (if any) is described by a docstring. |
| prev_line_doc = module_doc |
| |
| # A list of comments that occur before or on the current |
| # logical line, used to build the comment docstring. Each |
| # element is a tuple (comment_text, comment_lineno). |
| comments = [] |
| |
| # A list of decorator lines that occur before the current |
| # logical line. This is used so we can process a function |
| # declaration line and its decorators all at once. |
| decorators = [] |
| |
| # A list of group names, one for each indentation level. This is |
| # used to keep track groups that are defined by comment markers |
| # START_GROUP_MARKER and END_GROUP_MARKER. |
| groups = [None] |
| |
| # When we encounter a comment start group marker, set this to the |
| # name of the group; but wait until we're ready to process the |
| # next line before we actually set groups[-1] to this value. This |
| # is necessary because at the top of a block, the tokenizer gives |
| # us comments before the INDENT token; but if we encounter a group |
| # start marker at the top of a block, then we want it to apply |
| # inside that block, not outside it. |
| start_group = None |
| |
| # Check if the source file declares an encoding. |
| encoding = get_module_encoding(module_doc.filename) |
| |
| # The token-eating loop: |
| try: |
| module_file = codecs.open(module_doc.filename, 'rU', encoding) |
| except LookupError: |
| log.warning("Unknown encoding %r for %s; using the default" |
| "encoding instead (iso-8859-1)" % |
| (encoding, module_doc.filename)) |
| encoding = 'iso-8859-1' |
| module_file = codecs.open(module_doc.filename, 'rU', encoding) |
| tok_iter = tokenize.generate_tokens(module_file.readline) |
| for toktype, toktext, (srow,scol), (erow,ecol), line_str in tok_iter: |
| # BOM encoding marker: ignore. |
| if (toktype == token.ERRORTOKEN and |
| (toktext == u'\ufeff' or |
| toktext.encode(encoding) == '\xef\xbb\xbf')): |
| pass |
| |
| # Error token: abort |
| elif toktype == token.ERRORTOKEN: |
| raise ParseError('Error during parsing: invalid syntax ' |
| '(%s, line %d, char %d: %r)' % |
| (module_doc.filename, srow, scol, toktext)) |
| |
| # Indent token: update the parent_doc stack. |
| elif toktype == token.INDENT: |
| if prev_line_doc is None: |
| parent_docs.append(parent_docs[-1]) |
| else: |
| parent_docs.append(prev_line_doc) |
| groups.append(None) |
| |
| # Dedent token: update the parent_doc stack. |
| elif toktype == token.DEDENT: |
| if line_toks == []: |
| parent_docs.pop() |
| groups.pop() |
| else: |
| # This *should* only happen if the file ends on an |
| # indented line, with no final newline. |
| # (otherwise, this is the wrong thing to do.) |
| pass |
| |
| # Line-internal newline token: if we're still at the start of |
| # the logical line, and we've seen one or more comment lines, |
| # then discard them: blank lines are not allowed between a |
| # comment block and the thing it describes. |
| elif toktype == tokenize.NL: |
| if comments and not line_toks: |
| log.warning('Ignoring docstring comment block followed by ' |
| 'a blank line in %r on line %r' % |
| (module_doc.filename, srow-1)) |
| comments = [] |
| |
| # Comment token: add to comments if appropriate. |
| elif toktype == tokenize.COMMENT: |
| if toktext.startswith(COMMENT_DOCSTRING_MARKER): |
| comment_line = toktext[len(COMMENT_DOCSTRING_MARKER):].rstrip() |
| if comment_line.startswith(" "): |
| comment_line = comment_line[1:] |
| comments.append( [comment_line, srow]) |
| elif toktext.startswith(START_GROUP_MARKER): |
| start_group = toktext[len(START_GROUP_MARKER):].strip() |
| elif toktext.startswith(END_GROUP_MARKER): |
| for i in range(len(groups)-1, -1, -1): |
| if groups[i]: |
| groups[i] = None |
| break |
| else: |
| log.warning("Got group end marker without a corresponding " |
| "start marker in %r on line %r" % |
| (module_doc.filename, srow)) |
| |
| # Normal token: Add it to line_toks. (If it's a non-unicode |
| # string literal, then we need to re-encode using the file's |
| # encoding, to get back to the original 8-bit data; and then |
| # convert that string with 8-bit data to a 7-bit ascii |
| # representation.) |
| elif toktype != token.NEWLINE and toktype != token.ENDMARKER: |
| if lineno is None: lineno = srow |
| if toktype == token.STRING: |
| str_prefixes = re.match('[^\'"]*', toktext).group() |
| if 'u' not in str_prefixes: |
| s = toktext.encode(encoding) |
| toktext = decode_with_backslashreplace(s) |
| line_toks.append( (toktype, toktext) ) |
| |
| # Decorator line: add it to the decorators list. |
| elif line_toks and line_toks[0] == (token.OP, '@'): |
| decorators.append(shallow_parse(line_toks)) |
| line_toks = [] |
| |
| # End of line token, but nothing to do. |
| elif line_toks == []: |
| pass |
| |
| # End of line token: parse the logical line & process it. |
| else: |
| if start_group: |
| groups[-1] = start_group |
| start_group = None |
| |
| if parent_docs[-1] != 'skip_block': |
| try: |
| prev_line_doc = process_line( |
| shallow_parse(line_toks), parent_docs, prev_line_doc, |
| lineno, comments, decorators, encoding) |
| except ParseError, e: |
| raise ParseError('Error during parsing: invalid ' |
| 'syntax (%s, line %d) -- %s' % |
| (module_doc.filename, lineno, e)) |
| except KeyboardInterrupt, e: raise |
| except Exception, e: |
| log.error('Internal error during parsing (%s, line ' |
| '%s):\n%s' % (module_doc.filename, lineno, e)) |
| raise |
| |
| # grouping... |
| if groups[-1] and prev_line_doc not in (None, 'skip_block'): |
| if isinstance(prev_line_doc, VariableDoc): |
| # prev_line_doc's container will only be |
| # UNKNOWN if it's an instance variable that |
| # didn't have a doc-comment, but might still |
| # be followed by a docstring. Since we |
| # tokenize in order, we can't do lookahead to |
| # see if the variable will have a comment; but |
| # it should only be added to the container if |
| # it does. So we defer the grouping of that |
| # to be handled by process_docstring instead. |
| if prev_line_doc.container is not UNKNOWN: |
| add_to_group(prev_line_doc.container, |
| prev_line_doc, groups[-1]) |
| elif isinstance(parent_docs[-1], NamespaceDoc): |
| add_to_group(parent_docs[-1], prev_line_doc, |
| groups[-1]) |
| else: |
| prev_line_doc = None |
| |
| # Reset line contents. |
| line_toks = [] |
| lineno = None |
| comments = [] |
| decorators = [] |
| |
| def add_to_group(container, api_doc, group_name): |
| if container.group_specs is UNKNOWN: |
| container.group_specs = [] |
| |
| if isinstance(api_doc, VariableDoc): |
| var_name = api_doc.name |
| else: |
| if api_doc.canonical_name is UNKNOWN: log.debug('ouch', `api_doc`) |
| var_name = api_doc.canonical_name[-1] |
| |
| for (name, group_vars) in container.group_specs: |
| if name == group_name: |
| group_vars.append(var_name) |
| return |
| else: |
| container.group_specs.append( (group_name, [var_name]) ) |
| |
| def script_guard(line): |
| """Detect the idiomatic trick C{if __name__ == "__main__":}""" |
| return (len(line) == 5 |
| and line[1][1] == '__name__' # this is the most selective |
| and line[0][1] == 'if' |
| and line[2][1] == '==' |
| and line[4][1] == ':' |
| and line[3][1][1:-1] == '__main__') |
| |
| #///////////////////////////////////////////////////////////////// |
| #{ Shallow parser |
| #///////////////////////////////////////////////////////////////// |
| |
| def shallow_parse(line_toks): |
| """ |
| Given a flat list of tokens, return a nested tree structure |
| (called a X{token tree}), whose leaves are identical to the |
| original list, but whose structure reflects the structure |
| implied by the grouping tokens (i.e., parenthases, braces, and |
| brackets). If the parenthases, braces, and brackets do not |
| match, or are not balanced, then raise a ParseError. |
| |
| Assign some structure to a sequence of structure (group parens). |
| """ |
| stack = [[]] |
| parens = [] |
| for tok in line_toks: |
| toktype, toktext = tok |
| if toktext in ('(','[','{'): |
| parens.append(tok) |
| stack.append([tok]) |
| elif toktext in ('}',']',')'): |
| if not parens: |
| raise ParseError('Unbalanced parens') |
| left_paren = parens.pop()[1] |
| if left_paren+toktext not in ('()', '[]', '{}'): |
| raise ParseError('Mismatched parens') |
| lst = stack.pop() |
| lst.append(tok) |
| stack[-1].append(lst) |
| else: |
| stack[-1].append(tok) |
| if len(stack) != 1 or len(parens) != 0: |
| raise ParseError('Unbalanced parens') |
| return stack[0] |
| |
| #///////////////////////////////////////////////////////////////// |
| #{ Line processing |
| #///////////////////////////////////////////////////////////////// |
| # The methods process_*() are used to handle lines. |
| |
| def process_line(line, parent_docs, prev_line_doc, lineno, |
| comments, decorators, encoding): |
| """ |
| @return: C{new-doc}, C{decorator}..? |
| """ |
| args = (line, parent_docs, prev_line_doc, lineno, |
| comments, decorators, encoding) |
| |
| if not line: # blank line. |
| return None |
| elif (token.OP, ':') in line[:-1]: |
| return process_one_line_block(*args) |
| elif (token.OP, ';') in line: |
| return process_multi_stmt(*args) |
| elif line[0] == (token.NAME, 'def'): |
| return process_funcdef(*args) |
| elif line[0] == (token.OP, '@'): |
| return process_funcdef(*args) |
| elif line[0] == (token.NAME, 'class'): |
| return process_classdef(*args) |
| elif line[0] == (token.NAME, 'import'): |
| return process_import(*args) |
| elif line[0] == (token.NAME, 'from'): |
| return process_from_import(*args) |
| elif line[0] == (token.NAME, 'del'): |
| return process_del(*args) |
| elif len(line)==1 and line[0][0] == token.STRING: |
| return process_docstring(*args) |
| elif (token.OP, '=') in line: |
| return process_assignment(*args) |
| elif (line[0][0] == token.NAME and |
| line[0][1] in CONTROL_FLOW_KEYWORDS): |
| return process_control_flow_line(*args) |
| else: |
| return None |
| # [xx] do something with control structures like for/if? |
| |
| #///////////////////////////////////////////////////////////////// |
| # Line handler: control flow |
| #///////////////////////////////////////////////////////////////// |
| |
| CONTROL_FLOW_KEYWORDS = [ |
| #: A list of the control flow keywords. If a line begins with |
| #: one of these keywords, then it should be handled by |
| #: C{process_control_flow_line}. |
| 'if', 'elif', 'else', 'while', 'for', 'try', 'except', 'finally'] |
| |
| def process_control_flow_line(line, parent_docs, prev_line_doc, |
| lineno, comments, decorators, encoding): |
| keyword = line[0][1] |
| |
| # If it's a 'for' block: create the loop variable. |
| if keyword == 'for' and PARSE_FOR_BLOCKS: |
| loopvar_name = parse_dotted_name( |
| split_on(line[1:], (token.NAME, 'in'))[0]) |
| parent = get_lhs_parent(loopvar_name, parent_docs) |
| if parent is not None: |
| var_doc = VariableDoc(name=loopvar_name[-1], is_alias=False, |
| is_imported=False, is_instvar=False, |
| docs_extracted_by='parser') |
| set_variable(parent, var_doc) |
| |
| if ((keyword == 'if' and PARSE_IF_BLOCKS and not script_guard(line)) or |
| (keyword == 'elif' and PARSE_ELSE_BLOCKS) or |
| (keyword == 'else' and PARSE_ELSE_BLOCKS) or |
| (keyword == 'while' and PARSE_WHILE_BLOCKS) or |
| (keyword == 'for' and PARSE_FOR_BLOCKS) or |
| (keyword == 'try' and PARSE_TRY_BLOCKS) or |
| (keyword == 'except' and PARSE_EXCEPT_BLOCKS) or |
| (keyword == 'finally' and PARSE_FINALLY_BLOCKS)): |
| # Return "None" to indicate that we should process the |
| # block using the same context that we were already in. |
| return None |
| else: |
| # Return 'skip_block' to indicate that we should ignore |
| # the contents of this block. |
| return 'skip_block' |
| |
| #///////////////////////////////////////////////////////////////// |
| # Line handler: imports |
| #///////////////////////////////////////////////////////////////// |
| # [xx] I could optionally add ValueDoc's for the imported |
| # variables with proxy_for set to the imported source; but |
| # I don't think I gain much of anything by doing so. |
| |
| def process_import(line, parent_docs, prev_line_doc, lineno, |
| comments, decorators, encoding): |
| if not isinstance(parent_docs[-1], NamespaceDoc): return |
| |
| names = split_on(line[1:], (token.OP, ',')) |
| |
| for name in names: |
| name_pieces = split_on(name, (token.NAME, 'as')) |
| if len(name_pieces) == 1: |
| src_name = parse_dotted_name(name_pieces[0]) |
| _import_var(src_name, parent_docs) |
| elif len(name_pieces) == 2: |
| if len(name_pieces[1]) != 1: |
| raise ParseError('Expected identifier after "as"') |
| src_name = parse_dotted_name(name_pieces[0]) |
| var_name = parse_name(name_pieces[1][0]) |
| _import_var_as(src_name, var_name, parent_docs) |
| else: |
| raise ParseError('Multiple "as" tokens in import') |
| |
| def process_from_import(line, parent_docs, prev_line_doc, lineno, |
| comments, decorators, encoding): |
| if not isinstance(parent_docs[-1], NamespaceDoc): return |
| |
| pieces = split_on(line[1:], (token.NAME, 'import')) |
| if len(pieces) != 2 or not pieces[0] or not pieces[1]: |
| raise ParseError("Bad from-import") |
| lhs, rhs = pieces |
| |
| # The RHS might be parenthasized, as specified by PEP 328: |
| # http://www.python.org/peps/pep-0328.html |
| if (len(rhs) == 1 and isinstance(rhs[0], list) and |
| rhs[0][0] == (token.OP, '(') and rhs[0][-1] == (token.OP, ')')): |
| rhs = rhs[0][1:-1] |
| |
| # >>> from __future__ import nested_scopes |
| if lhs == [(token.NAME, '__future__')]: |
| return |
| |
| # >>> from sys import * |
| elif rhs == [(token.OP, '*')]: |
| src_name = parse_dotted_name(lhs) |
| _process_fromstar_import(src_name, parent_docs) |
| |
| # >>> from os.path import join, split |
| else: |
| # Allow relative imports in this case, as per PEP 328 |
| src_name = parse_dotted_name(lhs, |
| parent_name=parent_docs[-1].canonical_name) |
| parts = split_on(rhs, (token.OP, ',')) |
| for part in parts: |
| # from m import x |
| if len(part) == 1: |
| var_name = parse_name(part[0]) |
| _import_var_as(DottedName(src_name, var_name), |
| var_name, parent_docs) |
| |
| # from m import x as y |
| elif len(part) == 3 and part[1] == (token.NAME, 'as'): |
| orig_name = parse_name(part[0]) |
| var_name = parse_name(part[2]) |
| _import_var_as(DottedName(src_name, orig_name), |
| var_name, parent_docs) |
| |
| else: |
| ParseError("Bad from-import") |
| |
| def _process_fromstar_import(src, parent_docs): |
| """ |
| Handle a statement of the form: |
| >>> from <src> import * |
| |
| If L{IMPORT_HANDLING} is C{'parse'}, then first try to parse |
| the module C{M{<src>}}, and copy all of its exported variables |
| to C{parent_docs[-1]}. |
| |
| Otherwise, try to determine the names of the variables exported by |
| C{M{<src>}}, and create a new variable for each export. If |
| L{IMPORT_STAR_HANDLING} is C{'parse'}, then the list of exports if |
| found by parsing C{M{<src>}}; if it is C{'introspect'}, then the |
| list of exports is found by importing and introspecting |
| C{M{<src>}}. |
| """ |
| # This is redundant: already checked by caller. |
| if not isinstance(parent_docs[-1], NamespaceDoc): return |
| |
| # If src is package-local, then convert it to a global name. |
| src = _global_name(src, parent_docs) |
| |
| # Record the import |
| parent_docs[0].imports.append(src) # mark that it's .*?? |
| |
| # [xx] add check for if we already have the source docs in our |
| # cache?? |
| |
| if (IMPORT_HANDLING == 'parse' or |
| IMPORT_STAR_HANDLING == 'parse'): # [xx] is this ok? |
| try: module_doc = _find(src) |
| except ImportError: module_doc = None |
| if isinstance(module_doc, ModuleDoc): |
| for name, imp_var in module_doc.variables.items(): |
| # [xx] this is not exactly correct, but close. It |
| # does the wrong thing if a __var__ is explicitly |
| # listed in __all__. |
| if (imp_var.is_public and |
| not (name.startswith('__') and name.endswith('__'))): |
| var_doc = _add_import_var(DottedName(src, name), name, |
| parent_docs[-1]) |
| if IMPORT_HANDLING == 'parse': |
| var_doc.value = imp_var.value |
| |
| # If we got here, then either IMPORT_HANDLING='link' or we |
| # failed to parse the `src` module. |
| if IMPORT_STAR_HANDLING == 'introspect': |
| try: module = __import__(str(src), {}, {}, [0]) |
| except: return # We couldn't import it. |
| if module is None: return # We couldn't import it. |
| if hasattr(module, '__all__'): |
| names = list(module.__all__) |
| else: |
| names = [n for n in dir(module) if not n.startswith('_')] |
| for name in names: |
| _add_import_var(DottedName(src, name), name, parent_docs[-1]) |
| |
| def _import_var(name, parent_docs): |
| """ |
| Handle a statement of the form: |
| >>> import <name> |
| |
| If L{IMPORT_HANDLING} is C{'parse'}, then first try to find |
| the value by parsing; and create an appropriate variable in |
| parentdoc. |
| |
| Otherwise, add a variable for the imported variable. (More than |
| one variable may be created for cases like C{'import a.b'}, where |
| we need to create a variable C{'a'} in parentdoc containing a |
| proxy module; and a variable C{'b'} in the proxy module. |
| """ |
| # This is redundant: already checked by caller. |
| if not isinstance(parent_docs[-1], NamespaceDoc): return |
| |
| # If name is package-local, then convert it to a global name. |
| src = _global_name(name, parent_docs) |
| src_prefix = src[:len(src)-len(name)] |
| |
| # Record the import |
| parent_docs[0].imports.append(name) |
| |
| # [xx] add check for if we already have the source docs in our |
| # cache?? |
| |
| if IMPORT_HANDLING == 'parse': |
| # Check to make sure that we can actually find the value. |
| try: val_doc = _find(src) |
| except ImportError: val_doc = None |
| if val_doc is not None: |
| # We found it; but it's not the value itself we want to |
| # import, but the module containing it; so import that |
| # module (=top_mod) and create a variable for it. |
| top_mod = src_prefix+name[0] |
| var_doc = _add_import_var(top_mod, name[0], parent_docs[-1]) |
| var_doc.value = _find(DottedName(name[0])) |
| return |
| |
| # If we got here, then either IMPORT_HANDLING='link', or we |
| # did not successfully find the value's docs by parsing; use |
| # a variable with an UNKNOWN value. |
| |
| # Create any necessary intermediate proxy module values. |
| container = parent_docs[-1] |
| for i, identifier in enumerate(name[:-1]): |
| if (identifier not in container.variables or |
| not isinstance(container.variables[identifier], ModuleDoc)): |
| var_doc = _add_import_var(name[:i+1], identifier, container) |
| var_doc.value = ModuleDoc(variables={}, sort_spec=[], |
| proxy_for=src_prefix+name[:i+1], |
| submodules={}, |
| docs_extracted_by='parser') |
| container = container.variables[identifier].value |
| |
| # Add the variable to the container. |
| _add_import_var(src, name[-1], container) |
| |
| def _import_var_as(src, name, parent_docs): |
| """ |
| Handle a statement of the form: |
| >>> import src as name |
| |
| If L{IMPORT_HANDLING} is C{'parse'}, then first try to find |
| the value by parsing; and create an appropriate variable in |
| parentdoc. |
| |
| Otherwise, create a variables with its C{imported_from} attribute |
| pointing to the imported object. |
| """ |
| # This is redundant: already checked by caller. |
| if not isinstance(parent_docs[-1], NamespaceDoc): return |
| |
| # If src is package-local, then convert it to a global name. |
| src = _global_name(src, parent_docs) |
| |
| # Record the import |
| parent_docs[0].imports.append(src) |
| |
| if IMPORT_HANDLING == 'parse': |
| # Parse the value and create a variable for it. |
| try: val_doc = _find(src) |
| except ImportError: val_doc = None |
| if val_doc is not None: |
| var_doc = VariableDoc(name=name, value=val_doc, |
| is_imported=True, is_alias=False, |
| imported_from=src, |
| docs_extracted_by='parser') |
| set_variable(parent_docs[-1], var_doc) |
| return |
| |
| # If we got here, then either IMPORT_HANDLING='link', or we |
| # did not successfully find the value's docs by parsing; use a |
| # variable with a proxy value. |
| _add_import_var(src, name, parent_docs[-1]) |
| |
| def _add_import_var(src, name, container): |
| """ |
| Add a new imported variable named C{name} to C{container}, with |
| C{imported_from=src}. |
| """ |
| var_doc = VariableDoc(name=name, is_imported=True, is_alias=False, |
| imported_from=src, docs_extracted_by='parser') |
| set_variable(container, var_doc) |
| return var_doc |
| |
| def _global_name(name, parent_docs): |
| """ |
| If the given name is package-local (relative to the current |
| context, as determined by C{parent_docs}), then convert it |
| to a global name. |
| """ |
| # Get the containing package from parent_docs. |
| if parent_docs[0].is_package: |
| package = parent_docs[0] |
| else: |
| package = parent_docs[0].package |
| |
| # Check each package (from closest to furthest) to see if it |
| # contains a module named name[0]; if so, then treat `name` as |
| # relative to that package. |
| while package not in (None, UNKNOWN): |
| try: |
| fp = imp.find_module(name[0], package.path)[0] |
| if fp is not None: fp.close() |
| except ImportError: |
| # No submodule found here; try the next package up. |
| package = package.package |
| continue |
| # A submodule was found; return its name. |
| return package.canonical_name + name |
| |
| # We didn't find any package containing `name`; so just return |
| # `name` as-is. |
| return name |
| |
| #///////////////////////////////////////////////////////////////// |
| # Line handler: assignment |
| #///////////////////////////////////////////////////////////////// |
| |
| def process_assignment(line, parent_docs, prev_line_doc, lineno, |
| comments, decorators, encoding): |
| # Divide the assignment statement into its pieces. |
| pieces = split_on(line, (token.OP, '=')) |
| |
| lhs_pieces = pieces[:-1] |
| rhs = pieces[-1] |
| |
| # Decide whether the variable is an instance variable or not. |
| # If it's an instance var, then discard the value. |
| is_instvar = lhs_is_instvar(lhs_pieces, parent_docs) |
| |
| # if it's not an instance var, and we're not in a namespace, |
| # then it's just a local var -- so ignore it. |
| if not (is_instvar or isinstance(parent_docs[-1], NamespaceDoc)): |
| return None |
| |
| # Evaluate the right hand side. |
| if not is_instvar: |
| rhs_val, is_alias = rhs_to_valuedoc(rhs, parent_docs) |
| else: |
| rhs_val, is_alias = UNKNOWN, False |
| |
| # Assign the right hand side value to each left hand side. |
| # (Do the rightmost assignment first) |
| lhs_pieces.reverse() |
| for lhs in lhs_pieces: |
| # Try treating the LHS as a simple dotted name. |
| try: lhs_name = parse_dotted_name(lhs) |
| except: lhs_name = None |
| if lhs_name is not None: |
| lhs_parent = get_lhs_parent(lhs_name, parent_docs) |
| if lhs_parent is None: continue |
| |
| # Skip a special class variable. |
| if lhs_name[-1] == '__slots__': |
| continue |
| |
| # Create the VariableDoc. |
| var_doc = VariableDoc(name=lhs_name[-1], value=rhs_val, |
| is_imported=False, is_alias=is_alias, |
| is_instvar=is_instvar, |
| docs_extracted_by='parser') |
| # Extract a docstring from the comments, when present, |
| # but only if there's a single LHS. |
| if len(lhs_pieces) == 1: |
| add_docstring_from_comments(var_doc, comments) |
| |
| # Assign the variable to the containing namespace, |
| # *unless* the variable is an instance variable |
| # without a comment docstring. In that case, we'll |
| # only want to add it if we later discover that it's |
| # followed by a variable docstring. If it is, then |
| # process_docstring will take care of adding it to the |
| # containing clas. (This is a little hackish, but |
| # unfortunately is necessary because we won't know if |
| # this assignment line is followed by a docstring |
| # until later.) |
| if (not is_instvar) or comments: |
| set_variable(lhs_parent, var_doc, True) |
| |
| # If it's the only var, then return the VarDoc for use |
| # as the new `prev_line_doc`. |
| if (len(lhs_pieces) == 1 and |
| (len(lhs_name) == 1 or is_instvar)): |
| return var_doc |
| |
| # Otherwise, the LHS must be a complex expression; use |
| # dotted_names_in() to decide what variables it contains, |
| # and create VariableDoc's for all of them (with UNKNOWN |
| # value). |
| else: |
| for lhs_name in dotted_names_in(lhs_pieces): |
| lhs_parent = get_lhs_parent(lhs_name, parent_docs) |
| if lhs_parent is None: continue |
| var_doc = VariableDoc(name=lhs_name[-1], |
| is_imported=False, |
| is_alias=is_alias, |
| is_instvar=is_instvar, |
| docs_extracted_by='parser') |
| set_variable(lhs_parent, var_doc, True) |
| |
| # If we have multiple left-hand-sides, then all but the |
| # rightmost one are considered aliases. |
| is_alias = True |
| |
| |
| def lhs_is_instvar(lhs_pieces, parent_docs): |
| if not isinstance(parent_docs[-1], RoutineDoc): |
| return False |
| # make sure that lhs_pieces is <self>.<name>, where <self> is |
| # the name of the first arg to the containing routinedoc, and |
| # <name> is a simple name. |
| posargs = parent_docs[-1].posargs |
| if posargs is UNKNOWN: return False |
| if not (len(lhs_pieces)==1 and len(posargs) > 0 and |
| len(lhs_pieces[0]) == 3 and |
| lhs_pieces[0][0] == (token.NAME, posargs[0]) and |
| lhs_pieces[0][1] == (token.OP, '.') and |
| lhs_pieces[0][2][0] == token.NAME): |
| return False |
| # Make sure we're in an instance method, and not a |
| # module-level function. |
| for i in range(len(parent_docs)-1, -1, -1): |
| if isinstance(parent_docs[i], ClassDoc): |
| return True |
| elif parent_docs[i] != parent_docs[-1]: |
| return False |
| return False |
| |
| def rhs_to_valuedoc(rhs, parent_docs): |
| # Dotted variable: |
| try: |
| rhs_name = parse_dotted_name(rhs) |
| rhs_val = lookup_value(rhs_name, parent_docs) |
| if rhs_val is not None and rhs_val is not UNKNOWN: |
| return rhs_val, True |
| except ParseError: |
| pass |
| |
| # Decorators: |
| if (len(rhs)==2 and rhs[0][0] == token.NAME and |
| isinstance(rhs[1], list)): |
| arg_val, _ = rhs_to_valuedoc(rhs[1][1:-1], parent_docs) |
| if isinstance(arg_val, RoutineDoc): |
| doc = apply_decorator(DottedName(rhs[0][1]), arg_val) |
| doc.canonical_name = UNKNOWN |
| doc.parse_repr = pp_toktree(rhs) |
| return doc, False |
| |
| # Nothing else to do: make a val with the source as its repr. |
| return GenericValueDoc(parse_repr=pp_toktree(rhs), toktree=rhs, |
| defining_module=parent_docs[0], |
| docs_extracted_by='parser'), False |
| |
| def get_lhs_parent(lhs_name, parent_docs): |
| assert isinstance(lhs_name, DottedName) |
| |
| # For instance vars inside an __init__ method: |
| if isinstance(parent_docs[-1], RoutineDoc): |
| for i in range(len(parent_docs)-1, -1, -1): |
| if isinstance(parent_docs[i], ClassDoc): |
| return parent_docs[i] |
| else: |
| raise ValueError("%r is not a namespace or method" % |
| parent_docs[-1]) |
| |
| # For local variables: |
| if len(lhs_name) == 1: |
| return parent_docs[-1] |
| |
| # For non-local variables: |
| return lookup_value(lhs_name.container(), parent_docs) |
| |
| #///////////////////////////////////////////////////////////////// |
| # Line handler: single-line blocks |
| #///////////////////////////////////////////////////////////////// |
| |
| def process_one_line_block(line, parent_docs, prev_line_doc, lineno, |
| comments, decorators, encoding): |
| """ |
| The line handler for single-line blocks, such as: |
| |
| >>> def f(x): return x*2 |
| |
| This handler calls L{process_line} twice: once for the tokens |
| up to and including the colon, and once for the remaining |
| tokens. The comment docstring is applied to the first line |
| only. |
| @return: C{None} |
| """ |
| i = line.index((token.OP, ':')) |
| doc1 = process_line(line[:i+1], parent_docs, prev_line_doc, |
| lineno, comments, decorators, encoding) |
| doc2 = process_line(line[i+1:], parent_docs+[doc1], |
| doc1, lineno, None, [], encoding) |
| return doc1 |
| |
| #///////////////////////////////////////////////////////////////// |
| # Line handler: semicolon-separated statements |
| #///////////////////////////////////////////////////////////////// |
| |
| def process_multi_stmt(line, parent_docs, prev_line_doc, lineno, |
| comments, decorators, encoding): |
| """ |
| The line handler for semicolon-separated statements, such as: |
| |
| >>> x=1; y=2; z=3 |
| |
| This handler calls L{process_line} once for each statement. |
| The comment docstring is not passed on to any of the |
| sub-statements. |
| @return: C{None} |
| """ |
| for statement in split_on(line, (token.OP, ';')): |
| if not statement: continue |
| doc = process_line(statement, parent_docs, prev_line_doc, |
| lineno, None, decorators, encoding) |
| prev_line_doc = doc |
| decorators = [] |
| return None |
| |
| #///////////////////////////////////////////////////////////////// |
| # Line handler: delete statements |
| #///////////////////////////////////////////////////////////////// |
| |
| def process_del(line, parent_docs, prev_line_doc, lineno, |
| comments, decorators, encoding): |
| """ |
| The line handler for delete statements, such as: |
| |
| >>> del x, y.z |
| |
| This handler calls L{del_variable} for each dotted variable in |
| the variable list. The variable list may be nested. Complex |
| expressions in the variable list (such as C{x[3]}) are ignored. |
| @return: C{None} |
| """ |
| # If we're not in a namespace, then ignore it. |
| parent_doc = parent_docs[-1] |
| if not isinstance(parent_doc, NamespaceDoc): return |
| |
| var_list = split_on(line[1:], (token.OP, ',')) |
| for var_name in dotted_names_in(var_list): |
| del_variable(parent_docs[-1], var_name) |
| |
| return None |
| |
| #///////////////////////////////////////////////////////////////// |
| # Line handler: docstrings |
| #///////////////////////////////////////////////////////////////// |
| |
| def process_docstring(line, parent_docs, prev_line_doc, lineno, |
| comments, decorators, encoding): |
| """ |
| The line handler for bare string literals. If |
| C{prev_line_doc} is not C{None}, then the string literal is |
| added to that C{APIDoc} as a docstring. If it already has a |
| docstring (from comment docstrings), then the new docstring |
| will be appended to the old one. |
| """ |
| if prev_line_doc is None: return |
| docstring = parse_string(line) |
| |
| # If the docstring is a str, then convert it to unicode. |
| # According to a strict reading of PEP 263, this might not be the |
| # right thing to do; but it will almost always be what the |
| # module's author intended. |
| if isinstance(docstring, str): |
| try: |
| docstring = docstring.decode(encoding) |
| except UnicodeDecodeError: |
| # If decoding failed, then fall back on using |
| # decode_with_backslashreplace, which will map e.g. |
| # "\xe9" -> u"\\xe9". |
| docstring = decode_with_backslashreplace(docstring) |
| log.warning("While parsing %s: docstring is not a unicode " |
| "string, but it contains non-ascii data." % |
| prev_line_doc.canonical_name) |
| |
| # If the modified APIDoc is an instance variable, and it has |
| # not yet been added to its class's C{variables} list, |
| # then add it now. This is done here, rather than in the |
| # process_assignment() call that created the variable, because |
| # we only want to add instance variables if they have an |
| # associated docstring. (For more info, see the comment above |
| # the set_variable() call in process_assignment().) |
| added_instvar = False |
| if (isinstance(prev_line_doc, VariableDoc) and |
| prev_line_doc.is_instvar and |
| prev_line_doc.docstring in (None, UNKNOWN)): |
| for i in range(len(parent_docs)-1, -1, -1): |
| if isinstance(parent_docs[i], ClassDoc): |
| set_variable(parent_docs[i], prev_line_doc, True) |
| added_instvar = True |
| break |
| |
| if prev_line_doc.docstring not in (None, UNKNOWN): |
| log.warning("%s has both a comment-docstring and a normal " |
| "(string) docstring; ignoring the comment-" |
| "docstring." % prev_line_doc.canonical_name) |
| |
| prev_line_doc.docstring = docstring |
| prev_line_doc.docstring_lineno = lineno |
| |
| # If the modified APIDoc is an instance variable, and we added it |
| # to the class's variables list here, then it still needs to be |
| # grouped too; so return it for use as the new "prev_line_doc." |
| if added_instvar: |
| return prev_line_doc |
| |
| |
| #///////////////////////////////////////////////////////////////// |
| # Line handler: function declarations |
| #///////////////////////////////////////////////////////////////// |
| |
| def process_funcdef(line, parent_docs, prev_line_doc, lineno, |
| comments, decorators, encoding): |
| """ |
| The line handler for function declaration lines, such as: |
| |
| >>> def f(a, b=22, (c,d)): |
| |
| This handler creates and initializes a new C{VariableDoc} |
| containing a C{RoutineDoc}, adds the C{VariableDoc} to the |
| containing namespace, and returns the C{RoutineDoc}. |
| """ |
| # Check syntax. |
| if len(line) != 4 or line[3] != (token.OP, ':'): |
| raise ParseError("Bad function definition line") |
| |
| # If we're not in a namespace, then ignore it. |
| parent_doc = parent_docs[-1] |
| if not isinstance(parent_doc, NamespaceDoc): return |
| |
| # Get the function's name |
| func_name = parse_name(line[1]) |
| canonical_name = DottedName(parent_doc.canonical_name, func_name) |
| |
| # Create the function's RoutineDoc. |
| func_doc = RoutineDoc(canonical_name=canonical_name, |
| defining_module=parent_docs[0], |
| lineno=lineno, docs_extracted_by='parser') |
| |
| # Process the signature. |
| init_arglist(func_doc, line[2]) |
| |
| # If the preceeding comment includes a docstring, then add it. |
| add_docstring_from_comments(func_doc, comments) |
| |
| # Apply any decorators. |
| func_doc.decorators = [pp_toktree(deco[1:]) for deco in decorators] |
| decorators.reverse() |
| for decorator in decorators: |
| try: |
| deco_name = parse_dotted_name(decorator[1:]) |
| except ParseError: |
| deco_name = None |
| if func_doc.canonical_name is not UNKNOWN: |
| deco_repr = '%s(%s)' % (pp_toktree(decorator[1:]), |
| func_doc.canonical_name) |
| elif func_doc.parse_repr not in (None, UNKNOWN): |
| # [xx] this case should be improved.. when will func_doc |
| # have a known parse_repr?? |
| deco_repr = '%s(%s)' % (pp_toktree(decorator[1:]), |
| func_doc.parse_repr) |
| else: |
| deco_repr = UNKNOWN |
| func_doc = apply_decorator(deco_name, func_doc) |
| func_doc.parse_repr = deco_repr |
| # [XX] Is there a reson the following should be done? It |
| # causes the grouping code to break. Presumably the canonical |
| # name should remain valid if we're just applying a standard |
| # decorator. |
| #func_doc.canonical_name = UNKNOWN |
| |
| # Add a variable to the containing namespace. |
| var_doc = VariableDoc(name=func_name, value=func_doc, |
| is_imported=False, is_alias=False, |
| docs_extracted_by='parser') |
| set_variable(parent_doc, var_doc) |
| |
| # Return the new ValueDoc. |
| return func_doc |
| |
| def apply_decorator(decorator_name, func_doc): |
| # [xx] what if func_doc is not a RoutineDoc? |
| if decorator_name == DottedName('staticmethod'): |
| return StaticMethodDoc(**func_doc.__dict__) |
| elif decorator_name == DottedName('classmethod'): |
| return ClassMethodDoc(**func_doc.__dict__) |
| elif DEFAULT_DECORATOR_BEHAVIOR == 'transparent': |
| return func_doc.__class__(**func_doc.__dict__) # make a copy. |
| elif DEFAULT_DECORATOR_BEHAVIOR == 'opaque': |
| return GenericValueDoc(docs_extracted_by='parser') |
| else: |
| raise ValueError, 'Bad value for DEFAULT_DECORATOR_BEHAVIOR' |
| |
| def init_arglist(func_doc, arglist): |
| if not isinstance(arglist, list) or arglist[0] != (token.OP, '('): |
| raise ParseError("Bad argument list") |
| |
| # Initialize to defaults. |
| func_doc.posargs = [] |
| func_doc.posarg_defaults = [] |
| func_doc.vararg = None |
| func_doc.kwarg = None |
| |
| # Divide the arglist into individual args. |
| args = split_on(arglist[1:-1], (token.OP, ',')) |
| |
| # Keyword argument. |
| if args and args[-1][0] == (token.OP, '**'): |
| if len(args[-1]) != 2 or args[-1][1][0] != token.NAME: |
| raise ParseError("Expected name after ** in argument list") |
| func_doc.kwarg = args[-1][1][1] |
| args.pop() |
| |
| # Vararg argument. |
| if args and args[-1][0] == (token.OP, '*'): |
| if len(args[-1]) != 2 or args[-1][1][0] != token.NAME: |
| raise ParseError("Expected name after * in argument list") |
| func_doc.vararg = args[-1][1][1] |
| args.pop() |
| |
| # Positional arguments. |
| for arg in args: |
| func_doc.posargs.append(parse_funcdef_arg(arg[0])) |
| if len(arg) == 1: |
| func_doc.posarg_defaults.append(None) |
| elif arg[1] != (token.OP, '=') or len(arg) == 2: |
| raise ParseError("Bad argument list") |
| else: |
| default_repr = pp_toktree(arg[2:], 'tight') |
| default_val = GenericValueDoc(parse_repr=default_repr, |
| docs_extracted_by='parser') |
| func_doc.posarg_defaults.append(default_val) |
| |
| #///////////////////////////////////////////////////////////////// |
| # Line handler: class declarations |
| #///////////////////////////////////////////////////////////////// |
| |
| def process_classdef(line, parent_docs, prev_line_doc, lineno, |
| comments, decorators, encoding): |
| """ |
| The line handler for class declaration lines, such as: |
| |
| >>> class Foo(Bar, Baz): |
| |
| This handler creates and initializes a new C{VariableDoc} |
| containing a C{ClassDoc}, adds the C{VariableDoc} to the |
| containing namespace, and returns the C{ClassDoc}. |
| """ |
| # Check syntax |
| if len(line)<3 or len(line)>4 or line[-1] != (token.OP, ':'): |
| raise ParseError("Bad class definition line") |
| |
| # If we're not in a namespace, then ignore it. |
| parent_doc = parent_docs[-1] |
| if not isinstance(parent_doc, NamespaceDoc): return |
| |
| # Get the class's name |
| class_name = parse_name(line[1]) |
| canonical_name = DottedName(parent_doc.canonical_name, class_name) |
| |
| # Create the class's ClassDoc & VariableDoc. |
| class_doc = ClassDoc(variables={}, sort_spec=[], |
| bases=[], subclasses=[], |
| canonical_name=canonical_name, |
| defining_module=parent_docs[0], |
| docs_extracted_by='parser') |
| var_doc = VariableDoc(name=class_name, value=class_doc, |
| is_imported=False, is_alias=False, |
| docs_extracted_by='parser') |
| |
| # Add the bases. |
| if len(line) == 4: |
| if (not isinstance(line[2], list) or |
| line[2][0] != (token.OP, '(')): |
| raise ParseError("Expected base list") |
| try: |
| for base_name in parse_classdef_bases(line[2]): |
| class_doc.bases.append(find_base(base_name, parent_docs)) |
| except ParseError, e: |
| log.warning("Unable to extract the base list for %s: %s" % |
| (canonical_name, e)) |
| class_doc.bases = UNKNOWN |
| else: |
| class_doc.bases = [] |
| |
| # Register ourselves as a subclass to our bases. |
| if class_doc.bases is not UNKNOWN: |
| for basedoc in class_doc.bases: |
| if isinstance(basedoc, ClassDoc): |
| # This test avoids that a subclass gets listed twice when |
| # both introspection and parsing. |
| # [XXX] This check only works because currently parsing is |
| # always performed just after introspection of the same |
| # class. A more complete fix shuld be independent from |
| # calling order; probably the subclasses list should be |
| # replaced by a ClassDoc set or a {name: ClassDoc} mapping. |
| if (basedoc.subclasses |
| and basedoc.subclasses[-1].canonical_name |
| != class_doc.canonical_name): |
| basedoc.subclasses.append(class_doc) |
| |
| # If the preceeding comment includes a docstring, then add it. |
| add_docstring_from_comments(class_doc, comments) |
| |
| # Add the VariableDoc to our container. |
| set_variable(parent_doc, var_doc) |
| |
| return class_doc |
| |
| def _proxy_base(**attribs): |
| return ClassDoc(variables={}, sort_spec=[], bases=[], subclasses=[], |
| docs_extracted_by='parser', **attribs) |
| |
| def find_base(name, parent_docs): |
| assert isinstance(name, DottedName) |
| |
| # Find the variable containing the base. |
| base_var = lookup_variable(name, parent_docs) |
| if base_var is None: |
| # If we didn't find it, then it must have been imported. |
| # First, check if it looks like it's contained in any |
| # known imported variable: |
| if len(name) > 1: |
| src = lookup_name(name[0], parent_docs) |
| if (src is not None and |
| src.imported_from not in (None, UNKNOWN)): |
| base_src = DottedName(src.imported_from, name[1:]) |
| base_var = VariableDoc(name=name[-1], is_imported=True, |
| is_alias=False, imported_from=base_src, |
| docs_extracted_by='parser') |
| # Otherwise, it must have come from an "import *" statement |
| # (or from magic, such as direct manipulation of the module's |
| # dictionary), so we don't know where it came from. So |
| # there's nothing left but to use an empty proxy. |
| if base_var is None: |
| return _proxy_base(parse_repr=str(name)) |
| #raise ParseError("Could not find %s" % name) |
| |
| # If the variable has a value, return that value. |
| if base_var.value is not UNKNOWN: |
| return base_var.value |
| |
| # Otherwise, if BASE_HANDLING is 'parse', try parsing the docs for |
| # the base class; if that fails, or if BASE_HANDLING is 'link', |
| # just make a proxy object. |
| if base_var.imported_from not in (None, UNKNOWN): |
| if BASE_HANDLING == 'parse': |
| old_sys_path = sys.path |
| try: |
| dirname = os.path.split(parent_docs[0].filename)[0] |
| sys.path = [dirname] + sys.path |
| try: |
| return parse_docs(name=str(base_var.imported_from)) |
| except ParseError: |
| log.info('Unable to parse base', base_var.imported_from) |
| except ImportError: |
| log.info('Unable to find base', base_var.imported_from) |
| finally: |
| sys.path = old_sys_path |
| |
| # Either BASE_HANDLING='link' or parsing the base class failed; |
| # return a proxy value for the base class. |
| return _proxy_base(proxy_for=base_var.imported_from) |
| else: |
| return _proxy_base(parse_repr=str(name)) |
| |
| #///////////////////////////////////////////////////////////////// |
| #{ Parsing |
| #///////////////////////////////////////////////////////////////// |
| |
| def dotted_names_in(elt_list): |
| """ |
| Return a list of all simple dotted names in the given |
| expression. |
| """ |
| names = [] |
| while elt_list: |
| elt = elt_list.pop() |
| if len(elt) == 1 and isinstance(elt[0], list): |
| # Nested list: process the contents |
| elt_list.extend(split_on(elt[0][1:-1], (token.OP, ','))) |
| else: |
| try: |
| names.append(parse_dotted_name(elt)) |
| except ParseError: |
| pass # complex expression -- ignore |
| return names |
| |
| def parse_name(elt, strip_parens=False): |
| """ |
| If the given token tree element is a name token, then return |
| that name as a string. Otherwise, raise ParseError. |
| @param strip_parens: If true, then if elt is a single name |
| enclosed in parenthases, then return that name. |
| """ |
| if strip_parens and isinstance(elt, list): |
| while (isinstance(elt, list) and len(elt) == 3 and |
| elt[0] == (token.OP, '(') and |
| elt[-1] == (token.OP, ')')): |
| elt = elt[1] |
| if isinstance(elt, list) or elt[0] != token.NAME: |
| raise ParseError("Bad name") |
| return elt[1] |
| |
| def parse_dotted_name(elt_list, strip_parens=True, parent_name=None): |
| """ |
| @param parent_name: canonical name of referring module, to resolve |
| relative imports. |
| @type parent_name: L{DottedName} |
| @bug: does not handle 'x.(y).z' |
| """ |
| if len(elt_list) == 0: raise ParseError("Bad dotted name") |
| |
| # Handle ((x.y).z). (If the contents of the parens include |
| # anything other than dotted names, such as (x,y), then we'll |
| # catch it below and raise a ParseError. |
| while (isinstance(elt_list[0], list) and |
| len(elt_list[0]) >= 3 and |
| elt_list[0][0] == (token.OP, '(') and |
| elt_list[0][-1] == (token.OP, ')')): |
| elt_list[:1] = elt_list[0][1:-1] |
| |
| # Convert a relative import into an absolute name. |
| prefix_name = None |
| if parent_name is not None and elt_list[0][-1] == '.': |
| items = 1 |
| while len(elt_list) > items and elt_list[items][-1] == '.': |
| items += 1 |
| |
| elt_list = elt_list[items:] |
| prefix_name = parent_name[:-items] |
| |
| # >>> from . import foo |
| if not elt_list: |
| if prefix_name == []: |
| raise ParseError("Attempted relative import in non-package, " |
| "or beyond toplevel package") |
| return prefix_name |
| |
| if len(elt_list) % 2 != 1: raise ParseError("Bad dotted name") |
| name = DottedName(parse_name(elt_list[0], True)) |
| if prefix_name is not None: |
| name = prefix_name + name |
| |
| for i in range(2, len(elt_list), 2): |
| dot, identifier = elt_list[i-1], elt_list[i] |
| if dot != (token.OP, '.'): |
| raise ParseError("Bad dotted name") |
| name = DottedName(name, parse_name(identifier, True)) |
| return name |
| |
| def split_on(elt_list, split_tok): |
| # [xx] add code to guarantee each elt is non-empty. |
| result = [[]] |
| for elt in elt_list: |
| if elt == split_tok: |
| if result[-1] == []: raise ParseError("Empty element from split") |
| result.append([]) |
| else: |
| result[-1].append(elt) |
| if result[-1] == []: result.pop() |
| return result |
| |
| def parse_funcdef_arg(elt): |
| """ |
| If the given tree token element contains a valid function |
| definition argument (i.e., an identifier token or nested list |
| of identifiers), then return a corresponding string identifier |
| or nested list of string identifiers. Otherwise, raise a |
| ParseError. |
| """ |
| if isinstance(elt, list): |
| if elt[0] == (token.OP, '('): |
| if len(elt) == 3: |
| return parse_funcdef_arg(elt[1]) |
| else: |
| return [parse_funcdef_arg(e) |
| for e in elt[1:-1] |
| if e != (token.OP, ',')] |
| else: |
| raise ParseError("Bad argument -- expected name or tuple") |
| elif elt[0] == token.NAME: |
| return elt[1] |
| else: |
| raise ParseError("Bad argument -- expected name or tuple") |
| |
| def parse_classdef_bases(elt): |
| """ |
| If the given tree token element contains a valid base list |
| (that contains only dotted names), then return a corresponding |
| list of L{DottedName}s. Otherwise, raise a ParseError. |
| |
| @bug: Does not handle either of:: |
| - class A( (base.in.parens) ): pass |
| - class B( (lambda:calculated.base)() ): pass |
| """ |
| if (not isinstance(elt, list) or |
| elt[0] != (token.OP, '(')): |
| raise ParseError("Bad base list") |
| |
| return [parse_dotted_name(n) |
| for n in split_on(elt[1:-1], (token.OP, ','))] |
| |
| # Used by: base list; 'del'; ... |
| def parse_dotted_name_list(elt_list): |
| """ |
| If the given list of tree token elements contains a |
| comma-separated list of dotted names, then return a |
| corresponding list of L{DottedName} objects. Otherwise, raise |
| ParseError. |
| """ |
| names = [] |
| |
| state = 0 |
| for elt in elt_list: |
| # State 0 -- Expecting a name, or end of arglist |
| if state == 0: |
| # Make sure it's a name |
| if isinstance(elt, tuple) and elt[0] == token.NAME: |
| names.append(DottedName(elt[1])) |
| state = 1 |
| else: |
| raise ParseError("Expected a name") |
| # State 1 -- Expecting comma, period, or end of arglist |
| elif state == 1: |
| if elt == (token.OP, '.'): |
| state = 2 |
| elif elt == (token.OP, ','): |
| state = 0 |
| else: |
| raise ParseError("Expected '.' or ',' or end of list") |
| # State 2 -- Continuation of dotted name. |
| elif state == 2: |
| if isinstance(elt, tuple) and elt[0] == token.NAME: |
| names[-1] = DottedName(names[-1], elt[1]) |
| state = 1 |
| else: |
| raise ParseError("Expected a name") |
| if state == 2: |
| raise ParseError("Expected a name") |
| return names |
| |
| def parse_string(elt_list): |
| if len(elt_list) == 1 and elt_list[0][0] == token.STRING: |
| # [xx] use something safer here? But it needs to deal with |
| # any string type (eg r"foo\bar" etc). |
| return eval(elt_list[0][1]) |
| else: |
| raise ParseError("Expected a string") |
| |
| # ['1', 'b', 'c'] |
| def parse_string_list(elt_list): |
| if (len(elt_list) == 1 and isinstance(elt_list, list) and |
| elt_list[0][0][1] in ('(', '[')): |
| elt_list = elt_list[0][1:-1] |
| |
| string_list = [] |
| for string_elt in split_on(elt_list, (token.OP, ',')): |
| string_list.append(parse_string(string_elt)) |
| |
| return string_list |
| |
| #///////////////////////////////////////////////////////////////// |
| #{ Variable Manipulation |
| #///////////////////////////////////////////////////////////////// |
| |
| def set_variable(namespace, var_doc, preserve_docstring=False): |
| """ |
| Add var_doc to namespace. If namespace already contains a |
| variable with the same name, then discard the old variable. If |
| C{preserve_docstring} is true, then keep the old variable's |
| docstring when overwriting a variable. |
| """ |
| # Choose which dictionary we'll be storing the variable in. |
| if not isinstance(namespace, NamespaceDoc): |
| return |
| |
| # This happens when the class definition has not been parsed, e.g. in |
| # sf bug #1693253 on ``Exception.x = y`` |
| if namespace.sort_spec is UNKNOWN: |
| namespace.sort_spec = namespace.variables.keys() |
| |
| # If we already have a variable with this name, then remove the |
| # old VariableDoc from the sort_spec list; and if we gave its |
| # value a canonical name, then delete it. |
| if var_doc.name in namespace.variables: |
| namespace.sort_spec.remove(var_doc.name) |
| old_var_doc = namespace.variables[var_doc.name] |
| if (old_var_doc.is_alias == False and |
| old_var_doc.value is not UNKNOWN): |
| old_var_doc.value.canonical_name = UNKNOWN |
| if (preserve_docstring and var_doc.docstring in (None, UNKNOWN) and |
| old_var_doc.docstring not in (None, UNKNOWN)): |
| var_doc.docstring = old_var_doc.docstring |
| var_doc.docstring_lineno = old_var_doc.docstring_lineno |
| # Add the variable to the namespace. |
| namespace.variables[var_doc.name] = var_doc |
| namespace.sort_spec.append(var_doc.name) |
| assert var_doc.container is UNKNOWN |
| var_doc.container = namespace |
| |
| def del_variable(namespace, name): |
| if not isinstance(namespace, NamespaceDoc): |
| return |
| |
| if name[0] in namespace.variables: |
| if len(name) == 1: |
| var_doc = namespace.variables[name[0]] |
| namespace.sort_spec.remove(name[0]) |
| del namespace.variables[name[0]] |
| if not var_doc.is_alias and var_doc.value is not UNKNOWN: |
| var_doc.value.canonical_name = UNKNOWN |
| else: |
| del_variable(namespace.variables[name[0]].value, name[1:]) |
| |
| #///////////////////////////////////////////////////////////////// |
| #{ Name Lookup |
| #///////////////////////////////////////////////////////////////// |
| |
| def lookup_name(identifier, parent_docs): |
| """ |
| Find and return the documentation for the variable named by |
| the given identifier. |
| |
| @rtype: L{VariableDoc} or C{None} |
| """ |
| # We need to check 3 namespaces: locals, globals, and builtins. |
| # Note that this is true even if we're in a version of python with |
| # nested scopes, because nested scope lookup does not apply to |
| # nested class definitions, and we're not worried about variables |
| # in nested functions. |
| if not isinstance(identifier, basestring): |
| raise TypeError('identifier must be a string') |
| |
| # Locals |
| if isinstance(parent_docs[-1], NamespaceDoc): |
| if identifier in parent_docs[-1].variables: |
| return parent_docs[-1].variables[identifier] |
| |
| # Globals (aka the containing module) |
| if isinstance(parent_docs[0], NamespaceDoc): |
| if identifier in parent_docs[0].variables: |
| return parent_docs[0].variables[identifier] |
| |
| # Builtins |
| builtins = epydoc.docintrospecter.introspect_docs(__builtin__) |
| if isinstance(builtins, NamespaceDoc): |
| if identifier in builtins.variables: |
| return builtins.variables[identifier] |
| |
| # We didn't find it; return None. |
| return None |
| |
| def lookup_variable(dotted_name, parent_docs): |
| assert isinstance(dotted_name, DottedName) |
| # If it's a simple identifier, use lookup_name. |
| if len(dotted_name) == 1: |
| return lookup_name(dotted_name[0], parent_docs) |
| |
| # If it's a dotted name with multiple pieces, look up the |
| # namespace containing the var (=parent) first; and then |
| # look for the var in that namespace. |
| else: |
| parent = lookup_value(dotted_name[:-1], parent_docs) |
| if (isinstance(parent, NamespaceDoc) and |
| dotted_name[-1] in parent.variables): |
| return parent.variables[dotted_name[-1]] |
| else: |
| return None # var not found. |
| |
| def lookup_value(dotted_name, parent_docs): |
| """ |
| Find and return the documentation for the value contained in |
| the variable with the given name in the current namespace. |
| """ |
| assert isinstance(dotted_name, DottedName) |
| var_doc = lookup_name(dotted_name[0], parent_docs) |
| |
| for i in range(1, len(dotted_name)): |
| if var_doc is None: return None |
| |
| if isinstance(var_doc.value, NamespaceDoc): |
| var_dict = var_doc.value.variables |
| elif (var_doc.value is UNKNOWN and |
| var_doc.imported_from not in (None, UNKNOWN)): |
| src_name = var_doc.imported_from + dotted_name[i:] |
| # [xx] do I want to create a proxy here?? |
| return GenericValueDoc(proxy_for=src_name, |
| parse_repr=str(dotted_name), |
| docs_extracted_by='parser') |
| else: |
| return None |
| |
| var_doc = var_dict.get(dotted_name[i]) |
| |
| if var_doc is None: return None |
| return var_doc.value |
| |
| #///////////////////////////////////////////////////////////////// |
| #{ Docstring Comments |
| #///////////////////////////////////////////////////////////////// |
| |
| def add_docstring_from_comments(api_doc, comments): |
| if api_doc is None or not comments: return |
| api_doc.docstring = '\n'.join([line for (line, lineno) in comments]) |
| api_doc.docstring_lineno = comments[0][1] |
| |
| #///////////////////////////////////////////////////////////////// |
| #{ Tree tokens |
| #///////////////////////////////////////////////////////////////// |
| |
| def _join_toktree(s1, s2): |
| # Join them. s1 = left side; s2 = right side. |
| if (s2=='' or s1=='' or |
| s1 in ('-','`') or s2 in ('}',']',')','`',':') or |
| s2[0] in ('.',',') or s1[-1] in ('(','[','{','.','\n',' ') or |
| (s2[0] == '(' and s1[-1] not in (',','='))): |
| return '%s%s' % (s1,s2) |
| elif (spacing=='tight' and |
| s1[-1] in '+-*/=,' or s2[0] in '+-*/=,'): |
| return '%s%s' % (s1, s2) |
| else: |
| return '%s %s' % (s1, s2) |
| |
| def _pp_toktree_add_piece(spacing, pieces, piece): |
| s1 = pieces[-1] |
| s2 = piece |
| |
| if (s2=='' or s1=='' or |
| s1 in ('-','`') or s2 in ('}',']',')','`',':') or |
| s2[0] in ('.',',') or s1[-1] in ('(','[','{','.','\n',' ') or |
| (s2[0] == '(' and s1[-1] not in (',','='))): |
| pass |
| elif (spacing=='tight' and |
| s1[-1] in '+-*/=,' or s2[0] in '+-*/=,'): |
| pass |
| else: |
| pieces.append(' ') |
| |
| pieces.append(piece) |
| |
| def pp_toktree(elts, spacing='normal', indent=0): |
| pieces = [''] |
| _pp_toktree(elts, spacing, indent, pieces) |
| return ''.join(pieces) |
| |
| def _pp_toktree(elts, spacing, indent, pieces): |
| add_piece = _pp_toktree_add_piece |
| |
| for elt in elts: |
| # Put a blank line before class & def statements. |
| if elt == (token.NAME, 'class') or elt == (token.NAME, 'def'): |
| add_piece(spacing, pieces, '\n%s' % (' '*indent)) |
| |
| if isinstance(elt, tuple): |
| if elt[0] == token.NEWLINE: |
| add_piece(spacing, pieces, ' '+elt[1]) |
| add_piece(spacing, pieces, '\n%s' % (' '*indent)) |
| elif elt[0] == token.INDENT: |
| add_piece(spacing, pieces, ' ') |
| indent += 1 |
| elif elt[0] == token.DEDENT: |
| assert pieces[-1] == ' ' |
| pieces.pop() |
| indent -= 1 |
| elif elt[0] == tokenize.COMMENT: |
| add_piece(spacing, pieces, elt[1].rstrip() + '\n') |
| add_piece(' '*indent) |
| else: |
| add_piece(spacing, pieces, elt[1]) |
| else: |
| _pp_toktree(elt, spacing, indent, pieces) |
| |
| #///////////////////////////////////////////////////////////////// |
| #{ Helper Functions |
| #///////////////////////////////////////////////////////////////// |
| |
| def get_module_encoding(filename): |
| """ |
| @see: U{PEP 263<http://www.python.org/peps/pep-0263.html>} |
| """ |
| module_file = open(filename, 'rU') |
| try: |
| lines = [module_file.readline() for i in range(2)] |
| if lines[0].startswith('\xef\xbb\xbf'): |
| return 'utf-8' |
| else: |
| for line in lines: |
| m = re.search("coding[:=]\s*([-\w.]+)", line) |
| if m: return m.group(1) |
| |
| # Fall back on Python's default encoding. |
| return 'iso-8859-1' # aka 'latin-1' |
| finally: |
| module_file.close() |
| |
| def _get_module_name(filename, package_doc): |
| """ |
| Return (dotted_name, is_package) |
| """ |
| name = re.sub(r'.py\w?$', '', os.path.split(filename)[1]) |
| if name == '__init__': |
| is_package = True |
| name = os.path.split(os.path.split(filename)[0])[1] |
| else: |
| is_package = False |
| |
| # [XX] if the module contains a script, then `name` may not |
| # necessarily be a valid identifier -- which will cause |
| # DottedName to raise an exception. Is that what I want? |
| if package_doc is None: |
| dotted_name = DottedName(name) |
| else: |
| dotted_name = DottedName(package_doc.canonical_name, name) |
| |
| # Check if the module looks like it's shadowed by a variable. |
| # If so, then add a "'" to the end of its canonical name, to |
| # distinguish it from the variable. |
| if package_doc is not None and name in package_doc.variables: |
| vardoc = package_doc.variables[name] |
| if (vardoc.value not in (None, UNKNOWN) and |
| vardoc.imported_from != dotted_name): |
| log.warning("Module %s might be shadowed by a variable with " |
| "the same name." % dotted_name) |
| dotted_name = DottedName(str(dotted_name)+"'") |
| |
| return dotted_name, is_package |
| |
| def flatten(lst, out=None): |
| """ |
| @return: a flat list containing the leaves of the given nested |
| list. |
| @param lst: The nested list that should be flattened. |
| """ |
| if out is None: out = [] |
| for elt in lst: |
| if isinstance(elt, (list, tuple)): |
| flatten(elt, out) |
| else: |
| out.append(elt) |
| return out |
| |
