cleanup of lexer
diff --git a/mako/lexer.py b/mako/lexer.py index 52a4b6d..65be795 100644 --- a/mako/lexer.py +++ b/mako/lexer.py
@@ -13,7 +13,9 @@ _regexp_cache = {} class Lexer(object): - def __init__(self, text, filename=None, disable_unicode=False, input_encoding=None, preprocessor=None): + def __init__(self, text, filename=None, + disable_unicode=False, + input_encoding=None, preprocessor=None): self.text = text self.filename = filename self.template = parsetree.TemplateNode(self.filename) @@ -31,8 +33,13 @@ self.preprocessor = [preprocessor] else: self.preprocessor = preprocessor - - exception_kwargs = property(lambda self:{'source':self.text, 'lineno':self.matched_lineno, 'pos':self.matched_charpos, 'filename':self.filename}) + + @property + def exception_kwargs(self): + return {'source':self.text, + 'lineno':self.matched_lineno, + 'pos':self.matched_charpos, + 'filename':self.filename} def match(self, regexp, flags=None): """match the given regular expression string and flags to the current text position. @@ -62,7 +69,8 @@ cp -=1 self.matched_charpos = mp - cp self.lineno += len(lines) - #print "MATCHED:", match.group(0), "LINE START:", self.matched_lineno, "LINE END:", self.lineno + #print "MATCHED:", match.group(0), "LINE START:", + # self.matched_lineno, "LINE END:", self.lineno #print "MATCH:", regexp, "\n", self.text[mp : mp + 15], (match and "TRUE" or "FALSE") return match @@ -76,15 +84,23 @@ if match: m = self.match(r'.*?%s' % match.group(1), re.S) if not m: - raise exceptions.SyntaxException("Unmatched '%s'" % match.group(1), **self.exception_kwargs) + raise exceptions.SyntaxException( + "Unmatched '%s'" % + match.group(1), + **self.exception_kwargs) else: match = self.match(r'(%s)' % r'|'.join(text)) if match: - return (self.text[startpos:self.match_position-len(match.group(1))], match.group(1)) + return \ + self.text[startpos:self.match_position-len(match.group(1))],\ + match.group(1) else: match = self.match(r".*?(?=\"|\'|#|%s)" % r'|'.join(text), re.S) if not match: - raise exceptions.SyntaxException("Expected: %s" % ','.join(text), **self.exception_kwargs) + raise exceptions.SyntaxException( + "Expected: %s" % + ','.join(text), + **self.exception_kwargs) def append_node(self, nodecls, *args, **kwargs): kwargs.setdefault('source', self.text) @@ -105,8 +121,12 @@ self.control_line.pop() elif node.is_primary: self.control_line.append(node) - elif len(self.control_line) and not self.control_line[-1].is_ternary(node.keyword): - raise exceptions.SyntaxException("Keyword '%s' not a legal ternary for keyword '%s'" % (node.keyword, self.control_line[-1].keyword), **self.exception_kwargs) + elif len(self.control_line) and \ + not self.control_line[-1].is_ternary(node.keyword): + raise exceptions.SyntaxException( + "Keyword '%s' not a legal ternary for keyword '%s'" % + (node.keyword, self.control_line[-1].keyword), + **self.exception_kwargs) def escape_code(self, text): if not self.disable_unicode and self.encoding: @@ -122,22 +142,34 @@ parsed_encoding = 'utf-8' me = self.match_encoding() if me is not None and me != 'utf-8': - raise exceptions.CompileException("Found utf-8 BOM in file, with conflicting magic encoding comment of '%s'" % me, self.text.decode('utf-8', 'ignore'), 0, 0, self.filename) + raise exceptions.CompileException( + "Found utf-8 BOM in file, with conflicting " + "magic encoding comment of '%s'" % me, + self.text.decode('utf-8', 'ignore'), + 0, 0, self.filename) else: parsed_encoding = self.match_encoding() if parsed_encoding: self.encoding = parsed_encoding + if not self.disable_unicode and not isinstance(self.text, unicode): if self.encoding: try: self.text = self.text.decode(self.encoding) except UnicodeDecodeError, e: - raise exceptions.CompileException("Unicode decode operation of encoding '%s' failed" % self.encoding, self.text.decode('utf-8', 'ignore'), 0, 0, self.filename) + raise exceptions.CompileException( + "Unicode decode operation of encoding '%s' failed" % + self.encoding, + self.text.decode('utf-8', 'ignore'), + 0, 0, self.filename) else: try: self.text = self.text.decode() except UnicodeDecodeError, e: - raise exceptions.CompileException("Could not read template using encoding of 'ascii'. Did you forget a magic encoding comment?", self.text.decode('utf-8', 'ignore'), 0, 0, self.filename) + raise exceptions.CompileException( + "Could not read template using encoding of 'ascii'. " + "Did you forget a magic encoding comment?", + self.text.decode('utf-8', 'ignore'), 0, 0, self.filename) self.textlength = len(self.text) @@ -167,9 +199,15 @@ raise exceptions.CompileException("assertion failed") if len(self.tag): - raise exceptions.SyntaxException("Unclosed tag: <%%%s>" % self.tag[-1].keyword, **self.exception_kwargs) + raise exceptions.SyntaxException("Unclosed tag: <%%%s>" % + self.tag[-1].keyword, + **self.exception_kwargs) if len(self.control_line): - raise exceptions.SyntaxException("Unterminated control keyword: '%s'" % self.control_line[-1].keyword, self.text, self.control_line[-1].lineno, self.control_line[-1].pos, self.filename) + raise exceptions.SyntaxException("Unterminated control keyword: '%s'" % + self.control_line[-1].keyword, + self.text, + self.control_line[-1].lineno, + self.control_line[-1].pos, self.filename) return self.template def match_encoding(self): @@ -196,12 +234,12 @@ re.I | re.S | re.X) if match: - (keyword, attr, isend) = (match.group(1), match.group(2), match.group(3)) + keyword, attr, isend = match.group(1), match.group(2), match.group(3) self.keyword = keyword attributes = {} if attr: for att in re.findall(r"\s*(\w+)\s*=\s*(?:'([^']*)'|\"([^\"]*)\")", attr): - (key, val1, val2) = att + key, val1, val2 = att text = val1 or val2 text = text.replace('\r\n', '\n') attributes[key] = self.escape_code(text) @@ -212,7 +250,10 @@ if keyword == 'text': match = self.match(r'(.*?)(?=\</%text>)', re.S) if not match: - raise exceptions.SyntaxException("Unclosed tag: <%%%s>" % self.tag[-1].keyword, **self.exception_kwargs) + raise exceptions.SyntaxException( + "Unclosed tag: <%%%s>" % + self.tag[-1].keyword, + **self.exception_kwargs) self.append_node(parsetree.Text, match.group(1)) return self.match_tag_end() return True @@ -223,9 +264,15 @@ match = self.match(r'\</%[\t ]*(.+?)[\t ]*>') if match: if not len(self.tag): - raise exceptions.SyntaxException("Closing tag without opening tag: </%%%s>" % match.group(1), **self.exception_kwargs) + raise exceptions.SyntaxException( + "Closing tag without opening tag: </%%%s>" % + match.group(1), + **self.exception_kwargs) elif self.tag[-1].keyword != match.group(1): - raise exceptions.SyntaxException("Closing tag </%%%s> does not match tag: <%%%s>" % (match.group(1), self.tag[-1].keyword), **self.exception_kwargs) + raise exceptions.SyntaxException( + "Closing tag </%%%s> does not match tag: <%%%s>" % + (match.group(1), self.tag[-1].keyword), + **self.exception_kwargs) self.tag.pop() return True else: @@ -246,16 +293,18 @@ match = self.match(r""" (.*?) # anything, followed by: ( - (?<=\n)(?=[ \t]*(?=%|\#\#)) # an eval or line-based comment preceded by a consumed \n and whitespace + (?<=\n)(?=[ \t]*(?=%|\#\#)) # an eval or line-based + # comment preceded by a + # consumed newline and whitespace | - (?=\${) # an expression + (?=\${) # an expression | - (?=\#\*) # multiline comment + (?=\#\*) # multiline comment | (?=</?[%&]) # a substitution or block or call start or end - # - don't consume + # - don't consume | - (\\\r?\n) # an escaped newline - throw away + (\\\r?\n) # an escaped newline - throw away | \Z # end of string )""", re.X | re.S) @@ -270,10 +319,15 @@ def match_python_block(self): match = self.match(r"<%(!)?") if match: - (line, pos) = (self.matched_lineno, self.matched_charpos) - (text, end) = self.parse_until_text(r'%>') - text = adjust_whitespace(text) + "\n" # the trailing newline helps compiler.parse() not complain about indentation - self.append_node(parsetree.Code, self.escape_code(text), match.group(1)=='!', lineno=line, pos=pos) + line, pos = self.matched_lineno, self.matched_charpos + text, end = self.parse_until_text(r'%>') + # the trailing newline helps + # compiler.parse() not complain about indentation + text = adjust_whitespace(text) + "\n" + self.append_node( + parsetree.Code, + self.escape_code(text), + match.group(1)=='!', lineno=line, pos=pos) return True else: return False @@ -281,14 +335,17 @@ def match_expression(self): match = self.match(r"\${") if match: - (line, pos) = (self.matched_lineno, self.matched_charpos) - (text, end) = self.parse_until_text(r'\|', r'}') + line, pos = self.matched_lineno, self.matched_charpos + text, end = self.parse_until_text(r'\|', r'}') if end == '|': - (escapes, end) = self.parse_until_text(r'}') + escapes, end = self.parse_until_text(r'}') else: escapes = "" text = text.replace('\r\n', '\n') - self.append_node(parsetree.Expression, self.escape_code(text), escapes.strip(), lineno=line, pos=pos) + self.append_node( + parsetree.Expression, + self.escape_code(text), escapes.strip(), + lineno=line, pos=pos) return True else: return False @@ -301,15 +358,24 @@ if operator == '%': m2 = re.match(r'(end)?(\w+)\s*(.*)', text) if not m2: - raise exceptions.SyntaxException("Invalid control line: '%s'" % text, **self.exception_kwargs) - (isend, keyword) = m2.group(1, 2) + raise exceptions.SyntaxException( + "Invalid control line: '%s'" % + text, + **self.exception_kwargs) + isend, keyword = m2.group(1, 2) isend = (isend is not None) if isend: if not len(self.control_line): - raise exceptions.SyntaxException("No starting keyword '%s' for '%s'" % (keyword, text), **self.exception_kwargs) + raise exceptions.SyntaxException( + "No starting keyword '%s' for '%s'" % + (keyword, text), + **self.exception_kwargs) elif self.control_line[-1].keyword != keyword: - raise exceptions.SyntaxException("Keyword '%s' doesn't match keyword '%s'" % (text, self.control_line[-1].keyword), **self.exception_kwargs) + raise exceptions.SyntaxException( + "Keyword '%s' doesn't match keyword '%s'" % + (text, self.control_line[-1].keyword), + **self.exception_kwargs) self.append_node(parsetree.ControlLine, keyword, isend, self.escape_code(text)) else: self.append_node(parsetree.Comment, text)