- Lexer/Compile exceptions propigate throughout lexer/parsetree/ast
using a more portable **exception_kwargs collection
- added "source" member to the dict propigated to Lexer/Compile exceptions
- RichTraceback can access original template source as a unicode object
using either 'source' memebr on Lexer/Compile exception, or 'source'
property on ModuleInfo, fixes #37
- unit tests for #37
diff --git a/CHANGES b/CHANGES
index b7406c6..f3476a5 100644
--- a/CHANGES
+++ b/CHANGES
@@ -8,6 +8,8 @@
- if plain Python defs are used with <%call>, a decorator
@runtime.supports_callable exists to ensure that the "caller"
stack is properly handled for the def.
+- fix to RichTraceback and exception reporting to get template
+ source code as a unicode object #37
0.1.5
- AST expression generation - added in just about everything
diff --git a/lib/mako/ast.py b/lib/mako/ast.py
index db51a88..0d00f30 100644
--- a/lib/mako/ast.py
+++ b/lib/mako/ast.py
@@ -12,15 +12,15 @@
from StringIO import StringIO
import re
-def parse(code, mode, lineno, pos, filename):
+def parse(code, mode, **exception_kwargs):
try:
return compiler_parse(code, mode)
except SyntaxError, e:
- raise exceptions.SyntaxException("(%s) %s (%s)" % (e.__class__.__name__, str(e), repr(code[0:50])), lineno, pos, filename)
+ raise exceptions.SyntaxException("(%s) %s (%s)" % (e.__class__.__name__, str(e), repr(code[0:50])), **exception_kwargs)
class PythonCode(object):
"""represents information about a string containing Python code"""
- def __init__(self, code, lineno, pos, filename):
+ def __init__(self, code, **exception_kwargs):
self.code = code
# represents all identifiers which are assigned to at some point in the code
@@ -36,7 +36,7 @@
# - AST is less likely to break with version changes (for example, the behavior of co_names changed a little bit
# in python version 2.5)
if isinstance(code, basestring):
- expr = parse(code.lstrip(), "exec", lineno, pos, filename)
+ expr = parse(code.lstrip(), "exec", **exception_kwargs)
else:
expr = code
@@ -97,14 +97,14 @@
self._add_declared(alias)
else:
if mod == '*':
- raise exceptions.CompileException("'import *' is not supported, since all identifier names must be explicitly declared. Please use the form 'from <modulename> import <name1>, <name2>, ...' instead.", lineno, pos, filename)
+ raise exceptions.CompileException("'import *' is not supported, since all identifier names must be explicitly declared. Please use the form 'from <modulename> import <name1>, <name2>, ...' instead.", **exception_kwargs)
self._add_declared(mod)
f = FindIdentifiers()
visitor.walk(expr, f) #, walker=walker())
class ArgumentList(object):
"""parses a fragment of code as a comma-separated list of expressions"""
- def __init__(self, code, lineno, pos, filename):
+ def __init__(self, code, **exception_kwargs):
self.codeargs = []
self.args = []
self.declared_identifiers = util.Set()
@@ -112,7 +112,7 @@
class FindTuple(object):
def visitTuple(s, node, *args):
for n in node.nodes:
- p = PythonCode(n, lineno, pos, filename)
+ p = PythonCode(n, **exception_kwargs)
self.codeargs.append(p)
self.args.append(ExpressionGenerator(n).value())
self.declared_identifiers = self.declared_identifiers.union(p.declared_identifiers)
@@ -122,7 +122,7 @@
# if theres text and no trailing comma, insure its parsed
# as a tuple by adding a trailing comma
code += ","
- expr = parse(code, "exec", lineno, pos, filename)
+ expr = parse(code, "exec", **exception_kwargs)
else:
expr = code
@@ -138,10 +138,10 @@
except (MyException, e):
etc.
"""
- def __init__(self, code, lineno, pos, filename):
+ def __init__(self, code, **exception_kwargs):
m = re.match(r'^(\w+)(?:\s+(.*?))?:$', code.strip())
if not m:
- raise exceptions.CompileException("Fragment '%s' is not a partial control statement" % code, lineno, pos, filename)
+ raise exceptions.CompileException("Fragment '%s' is not a partial control statement" % code, **exception_kwargs)
(keyword, expr) = m.group(1,2)
if keyword in ['for','if', 'while']:
code = code + "pass"
@@ -152,8 +152,8 @@
elif keyword == 'except':
code = "try:pass\n" + code + "pass"
else:
- raise exceptions.CompileException("Unsupported control keyword: '%s'" % keyword, lineno, pos, filename)
- super(PythonFragment, self).__init__(code, lineno, pos, filename)
+ raise exceptions.CompileException("Unsupported control keyword: '%s'" % keyword, **exception_kwargs)
+ super(PythonFragment, self).__init__(code, **exception_kwargs)
class walker(visitor.ASTVisitor):
def dispatch(self, node, *args):
@@ -163,9 +163,9 @@
class FunctionDecl(object):
"""function declaration"""
- def __init__(self, code, lineno, pos, filename, allow_kwargs=True):
+ def __init__(self, code, allow_kwargs=True, **exception_kwargs):
self.code = code
- expr = parse(code, "exec", lineno, pos, filename)
+ expr = parse(code, "exec", **exception_kwargs)
class ParseFunc(object):
def visitFunction(s, node, *args):
self.funcname = node.name
@@ -177,9 +177,9 @@
f = ParseFunc()
visitor.walk(expr, f)
if not hasattr(self, 'funcname'):
- raise exceptions.CompileException("Code '%s' is not a function declaration" % code, lineno, pos, filename)
+ raise exceptions.CompileException("Code '%s' is not a function declaration" % code, **exception_kwargs)
if not allow_kwargs and self.kwargs:
- raise exceptions.CompileException("'**%s' keyword argument not allowed here" % self.argnames[-1], lineno, pos, filename)
+ raise exceptions.CompileException("'**%s' keyword argument not allowed here" % self.argnames[-1], **exception_kwargs)
def get_argument_expressions(self, include_defaults=True):
"""return the argument declarations of this FunctionDecl as a printable list."""
@@ -207,8 +207,8 @@
class FunctionArgs(FunctionDecl):
"""the argument portion of a function declaration"""
- def __init__(self, code, lineno, pos, filename, **kwargs):
- super(FunctionArgs, self).__init__("def ANON(%s):pass" % code, lineno, pos, filename, **kwargs)
+ def __init__(self, code, **kwargs):
+ super(FunctionArgs, self).__init__("def ANON(%s):pass" % code, **kwargs)
class ExpressionGenerator(object):
diff --git a/lib/mako/codegen.py b/lib/mako/codegen.py
index db520da..379cb4c 100644
--- a/lib/mako/codegen.py
+++ b/lib/mako/codegen.py
@@ -14,20 +14,21 @@
MAGIC_NUMBER = 2
-def compile(node, uri, filename=None, default_filters=None, buffer_filters=None, imports=None):
+def compile(node, uri, filename=None, default_filters=None, buffer_filters=None, imports=None, source_encoding=None):
"""generate module source code given a parsetree node, uri, and optional source filename"""
buf = util.FastEncodingBuffer()
printer = PythonPrinter(buf)
- _GenerateRenderMethod(printer, _CompileContext(uri, filename, default_filters, buffer_filters, imports), node)
+ _GenerateRenderMethod(printer, _CompileContext(uri, filename, default_filters, buffer_filters, imports, source_encoding), node)
return buf.getvalue()
class _CompileContext(object):
- def __init__(self, uri, filename, default_filters, buffer_filters, imports):
+ def __init__(self, uri, filename, default_filters, buffer_filters, imports, source_encoding):
self.uri = uri
self.filename = filename
self.default_filters = default_filters
self.buffer_filters = buffer_filters
self.imports = imports
+ self.source_encoding = source_encoding
class _GenerateRenderMethod(object):
"""a template visitor object which generates the full module source for a template."""
@@ -116,12 +117,13 @@
self.printer.writeline("_template_filename=%s" % repr(self.compiler.filename))
self.printer.writeline("_template_uri=%s" % repr(self.compiler.uri))
self.printer.writeline("_template_cache=cache.Cache(__name__, _modified_time)")
+ self.printer.writeline("_source_encoding=%s" % repr(self.compiler.source_encoding))
if self.compiler.imports:
buf = ''
for imp in self.compiler.imports:
buf += imp + "\n"
self.printer.writeline(imp)
- impcode = ast.PythonCode(buf, 0, 0, 'template defined imports')
+ impcode = ast.PythonCode(buf, source='', lineno=0, pos=0, filename='template defined imports')
else:
impcode = None
@@ -445,8 +447,7 @@
else:
x = e
e = locate_encode(e)
- if e is None:
- raise "der its none " + x
+ assert e is not None
target = "%s(%s)" % (e, target)
return target
diff --git a/lib/mako/exceptions.py b/lib/mako/exceptions.py
index 264068e..b855adb 100644
--- a/lib/mako/exceptions.py
+++ b/lib/mako/exceptions.py
@@ -20,18 +20,20 @@
else:
return " in file '%s' at line: %d char: %d" % (filename, lineno, pos)
class CompileException(MakoException):
- def __init__(self, message, lineno, pos, filename):
+ def __init__(self, message, source, lineno, pos, filename):
MakoException.__init__(self, message + _format_filepos(lineno, pos, filename))
self.lineno =lineno
self.pos = pos
self.filename = filename
+ self.source = source
class SyntaxException(MakoException):
- def __init__(self, message, lineno, pos, filename):
+ def __init__(self, message, source, lineno, pos, filename):
MakoException.__init__(self, message + _format_filepos(lineno, pos, filename))
self.lineno =lineno
self.pos = pos
self.filename = filename
+ self.source = source
class TemplateLookupException(MakoException):
pass
@@ -68,7 +70,8 @@
if self.error is None:
self.error = t
if isinstance(self.error, CompileException) or isinstance(self.error, SyntaxException):
- self.source = file(self.error.filename).read()
+ import mako.template
+ self.source = self.error.source
self.lineno = self.error.lineno
self._has_source = True
self.reverse_records = [r for r in self.records]
diff --git a/lib/mako/lexer.py b/lib/mako/lexer.py
index c871340..dafa10d 100644
--- a/lib/mako/lexer.py
+++ b/lib/mako/lexer.py
@@ -30,7 +30,9 @@
self.preprocessor = [preprocessor]
else:
self.preprocessor = preprocessor
-
+
+ exception_kwargs = property(lambda self:{'source':self.text, 'lineno':self.matched_lineno, 'pos':self.matched_charpos, 'filename':self.filename})
+
def match(self, regexp, flags=None):
"""match the given regular expression string and flags to the current text position.
@@ -73,7 +75,7 @@
if match:
m = self.match(r'.*?%s' % match.group(1), re.S)
if not m:
- raise exceptions.SyntaxException("Unmatched '%s'" % match.group(1), self.matched_lineno, self.matched_charpos, self.filename)
+ raise exceptions.SyntaxException("Unmatched '%s'" % match.group(1), **self.exception_kwargs)
else:
match = self.match(r'(%s)' % r'|'.join(text))
if match:
@@ -81,9 +83,10 @@
else:
match = self.match(r".*?(?=\"|\'|#|%s)" % r'|'.join(text), re.S)
if not match:
- raise exceptions.SyntaxException("Expected: %s" % ','.join(text), self.matched_lineno, self.matched_charpos, self.filename)
+ raise exceptions.SyntaxException("Expected: %s" % ','.join(text), **self.exception_kwargs)
def append_node(self, nodecls, *args, **kwargs):
+ kwargs.setdefault('source', self.text)
kwargs.setdefault('lineno', self.matched_lineno)
kwargs.setdefault('pos', self.matched_charpos)
kwargs['filename'] = self.filename
@@ -102,7 +105,7 @@
elif node.is_primary:
self.control_line.append(node)
elif len(self.control_line) and not self.control_line[-1].is_ternary(node.keyword):
- raise exceptions.SyntaxException("Keyword '%s' not a legal ternary for keyword '%s'" % (node.keyword, self.control_line[-1].keyword), self.matched_lineno, self.matched_charpos, self.filename)
+ raise exceptions.SyntaxException("Keyword '%s' not a legal ternary for keyword '%s'" % (node.keyword, self.control_line[-1].keyword), **self.exception_kwargs)
def escape_code(self, text):
if self.encoding:
@@ -121,12 +124,12 @@
try:
self.text = self.text.decode(self.encoding)
except UnicodeDecodeError, e:
- raise exceptions.CompileException("Unicode decode operation of encoding '%s' failed" % self.encoding, 0, 0, self.filename)
+ raise exceptions.CompileException("Unicode decode operation of encoding '%s' failed" % self.encoding, self.text.decode('utf-8', 'ignore'), 0, 0, self.filename)
else:
try:
self.text = self.text.decode()
except UnicodeDecodeError, e:
- raise exceptions.CompileException("Could not read template using encoding of 'ascii'. Did you forget a magic encoding comment?", 0, 0, self.filename)
+ raise exceptions.CompileException("Could not read template using encoding of 'ascii'. Did you forget a magic encoding comment?", self.text.decode('utf-8', 'ignore'), 0, 0, self.filename)
self.textlength = len(self.text)
@@ -156,9 +159,9 @@
raise exceptions.CompileException("assertion failed")
if len(self.tag):
- raise exceptions.SyntaxException("Unclosed tag: <%%%s>" % self.tag[-1].keyword, self.matched_lineno, self.matched_charpos, self.filename)
+ raise exceptions.SyntaxException("Unclosed tag: <%%%s>" % self.tag[-1].keyword, **self.exception_kwargs)
if len(self.control_line):
- raise exceptions.SyntaxException("Unterminated control keyword: '%s'" % self.control_line[-1].keyword, self.control_line[-1].lineno, self.control_line[-1].pos, self.filename)
+ raise exceptions.SyntaxException("Unterminated control keyword: '%s'" % self.control_line[-1].keyword, self.text, self.control_line[-1].lineno, self.control_line[-1].pos, self.filename)
return self.template
def match_encoding(self):
@@ -199,7 +202,7 @@
if keyword == 'text':
match = self.match(r'(.*?)(?=\</%text>)', re.S)
if not match:
- raise exceptions.SyntaxException("Unclosed tag: <%%%s>" % self.tag[-1].keyword, self.matched_lineno, self.matched_charpos, self.filename)
+ raise exceptions.SyntaxException("Unclosed tag: <%%%s>" % self.tag[-1].keyword, **self.exception_kwargs)
self.append_node(parsetree.Text, match.group(1))
return self.match_tag_end()
return True
@@ -210,9 +213,9 @@
match = self.match(r'\</%[\t ]*(.+?)[\t ]*>')
if match:
if not len(self.tag):
- raise exceptions.SyntaxException("Closing tag without opening tag: </%%%s>" % match.group(1), self.matched_lineno, self.matched_charpos, self.filename)
+ raise exceptions.SyntaxException("Closing tag without opening tag: </%%%s>" % match.group(1), **self.exception_kwargs)
elif self.tag[-1].keyword != match.group(1):
- raise exceptions.SyntaxException("Closing tag </%%%s> does not match tag: <%%%s>" % (match.group(1), self.tag[-1].keyword), self.matched_lineno, self.matched_charpos, self.filename)
+ raise exceptions.SyntaxException("Closing tag </%%%s> does not match tag: <%%%s>" % (match.group(1), self.tag[-1].keyword), **self.exception_kwargs)
self.tag.pop()
return True
else:
@@ -287,15 +290,15 @@
if operator == '%':
m2 = re.match(r'(end)?(\w+)\s*(.*)', text)
if not m2:
- raise exceptions.SyntaxException("Invalid control line: '%s'" % text, self.matched_lineno, self.matched_charpos, self.filename)
+ raise exceptions.SyntaxException("Invalid control line: '%s'" % text, **self.exception_kwargs)
(isend, keyword) = m2.group(1, 2)
isend = (isend is not None)
if isend:
if not len(self.control_line):
- raise exceptions.SyntaxException("No starting keyword '%s' for '%s'" % (keyword, text), self.matched_lineno, self.matched_charpos, self.filename)
+ raise exceptions.SyntaxException("No starting keyword '%s' for '%s'" % (keyword, text), **self.exception_kwargs)
elif self.control_line[-1].keyword != keyword:
- raise exceptions.SyntaxException("Keyword '%s' doesn't match keyword '%s'" % (text, self.control_line[-1].keyword), self.matched_lineno, self.matched_charpos, self.filename)
+ raise exceptions.SyntaxException("Keyword '%s' doesn't match keyword '%s'" % (text, self.control_line[-1].keyword), **self.exception_kwargs)
self.append_node(parsetree.ControlLine, keyword, isend, self.escape_code(text))
else:
self.append_node(parsetree.Comment, text)
diff --git a/lib/mako/parsetree.py b/lib/mako/parsetree.py
index ebed303..0c24e0b 100644
--- a/lib/mako/parsetree.py
+++ b/lib/mako/parsetree.py
@@ -11,10 +11,14 @@
class Node(object):
"""base class for a Node in the parse tree."""
- def __init__(self, lineno, pos, filename):
+ def __init__(self, source, lineno, pos, filename):
+ self.source = source
self.lineno = lineno
self.pos = pos
self.filename = filename
+
+ exception_kwargs = property(lambda self:{'source':self.source, 'lineno':self.lineno, 'pos':self.pos, 'filename':self.filename})
+
def get_children(self):
return []
def accept_visitor(self, visitor):
@@ -27,7 +31,7 @@
class TemplateNode(Node):
"""a 'container' node that stores the overall collection of nodes."""
def __init__(self, filename):
- super(TemplateNode, self).__init__(0, 0, filename)
+ super(TemplateNode, self).__init__('', 0, 0, filename)
self.nodes = []
self.page_attributes = {}
def get_children(self):
@@ -52,7 +56,7 @@
self._declared_identifiers = []
self._undeclared_identifiers = []
else:
- code = ast.PythonFragment(text, self.lineno, self.pos, self.filename)
+ code = ast.PythonFragment(text, **self.exception_kwargs)
(self._declared_identifiers, self._undeclared_identifiers) = (code.declared_identifiers, code.undeclared_identifiers)
def declared_identifiers(self):
return self._declared_identifiers
@@ -94,7 +98,7 @@
super(Code, self).__init__(**kwargs)
self.text = text
self.ismodule = ismodule
- self.code = ast.PythonCode(text, self.lineno, self.pos, self.filename)
+ self.code = ast.PythonCode(text, **self.exception_kwargs)
def declared_identifiers(self):
return self.code.declared_identifiers
def undeclared_identifiers(self):
@@ -124,8 +128,8 @@
super(Expression, self).__init__(**kwargs)
self.text = text
self.escapes = escapes
- self.escapes_code = ast.ArgumentList(escapes, self.lineno, self.pos, self.filename)
- self.code = ast.PythonCode(text, self.lineno, self.pos, self.filename)
+ self.escapes_code = ast.ArgumentList(escapes, **self.exception_kwargs)
+ self.code = ast.PythonCode(text, **self.exception_kwargs)
def declared_identifiers(self):
return []
def undeclared_identifiers(self):
@@ -145,7 +149,7 @@
try:
cls = _TagMeta._classmap[keyword]
except KeyError:
- raise exceptions.CompileException("No such tag: '%s'" % keyword, kwargs['lineno'], kwargs['pos'], kwargs['filename'])
+ raise exceptions.CompileException("No such tag: '%s'" % keyword, source=kwargs['source'], lineno=kwargs['lineno'], pos=kwargs['pos'], filename=kwargs['filename'])
return type.__call__(cls, keyword, attributes, **kwargs)
class Tag(Node):
@@ -179,7 +183,7 @@
self._parse_attributes(expressions, nonexpressions)
missing = [r for r in required if r not in self.parsed_attributes]
if len(missing):
- raise exceptions.CompileException("Missing attribute(s): %s" % ",".join([repr(m) for m in missing]), self.lineno, self.pos, self.filename)
+ raise exceptions.CompileException("Missing attribute(s): %s" % ",".join([repr(m) for m in missing]), **self.exception_kwargs)
self.parent = None
self.nodes = []
def is_root(self):
@@ -195,7 +199,7 @@
for x in re.split(r'(\${.+?})', self.attributes[key]):
m = re.match(r'^\${(.+?)}$', x)
if m:
- code = ast.PythonCode(m.group(1), self.lineno, self.pos, self.filename)
+ code = ast.PythonCode(m.group(1), **self.exception_kwargs)
undeclared_identifiers = undeclared_identifiers.union(code.undeclared_identifiers)
expr.append(m.group(1))
else:
@@ -204,10 +208,10 @@
self.parsed_attributes[key] = " + ".join(expr)
elif key in nonexpressions:
if re.search(r'${.+?}', self.attributes[key]):
- raise exceptions.CompileException("Attibute '%s' in tag '%s' does not allow embedded expressions" %(key, self.keyword), self.lineno, self.pos, self.filename)
+ raise exceptions.CompileException("Attibute '%s' in tag '%s' does not allow embedded expressions" %(key, self.keyword), **self.exception_kwargs)
self.parsed_attributes[key] = repr(self.attributes[key])
else:
- raise exceptions.CompileException("Invalid attribute for tag '%s': '%s'" %(self.keyword, key), self.lineno, self.pos, self.filename)
+ raise exceptions.CompileException("Invalid attribute for tag '%s': '%s'" %(self.keyword, key), **self.exception_kwargs)
self.expression_undeclared_identifiers = undeclared_identifiers
def declared_identifiers(self):
return []
@@ -220,7 +224,7 @@
__keyword__ = 'include'
def __init__(self, keyword, attributes, **kwargs):
super(IncludeTag, self).__init__(keyword, attributes, ('file', 'import', 'args'), (), ('file',), **kwargs)
- self.page_args = ast.PythonCode("__DUMMY(%s)" % attributes.get('args', ''), self.lineno, self.pos, self.filename)
+ self.page_args = ast.PythonCode("__DUMMY(%s)" % attributes.get('args', ''), **self.exception_kwargs)
def declared_identifiers(self):
return []
def undeclared_identifiers(self):
@@ -233,7 +237,7 @@
super(NamespaceTag, self).__init__(keyword, attributes, (), ('name','inheritable','file','import','module'), (), **kwargs)
self.name = attributes.get('name', '__anon_%s' % hex(abs(id(self))))
if not 'name' in attributes and not 'import' in attributes:
- raise exceptions.CompileException("'name' and/or 'import' attributes are required for <%namespace>", self.lineno, self.pos, self.filename)
+ raise exceptions.CompileException("'name' and/or 'import' attributes are required for <%namespace>", **self.exception_kwargs)
def declared_identifiers(self):
return []
@@ -241,7 +245,7 @@
__keyword__ = 'text'
def __init__(self, keyword, attributes, **kwargs):
super(TextTag, self).__init__(keyword, attributes, (), ('filter'), (), **kwargs)
- self.filter_args = ast.ArgumentList(attributes.get('filter', ''), self.lineno, self.pos, self.filename)
+ self.filter_args = ast.ArgumentList(attributes.get('filter', ''), **self.exception_kwargs)
class DefTag(Tag):
__keyword__ = 'def'
@@ -249,24 +253,24 @@
super(DefTag, self).__init__(keyword, attributes, ('buffered', 'cached', 'cache_key', 'cache_timeout', 'cache_type', 'cache_dir', 'cache_url'), ('name','filter'), ('name',), **kwargs)
name = attributes['name']
if re.match(r'^[\w_]+$',name):
- raise exceptions.CompileException("Missing parenthesis in %def", self.lineno, self.pos, self.filename)
- self.function_decl = ast.FunctionDecl("def " + name + ":pass", self.lineno, self.pos, self.filename)
+ raise exceptions.CompileException("Missing parenthesis in %def", **self.exception_kwargs)
+ self.function_decl = ast.FunctionDecl("def " + name + ":pass", **self.exception_kwargs)
self.name = self.function_decl.funcname
- self.filter_args = ast.ArgumentList(attributes.get('filter', ''), self.lineno, self.pos, self.filename)
+ self.filter_args = ast.ArgumentList(attributes.get('filter', ''), **self.exception_kwargs)
def declared_identifiers(self):
return self.function_decl.argnames
def undeclared_identifiers(self):
res = []
for c in self.function_decl.defaults:
- res += list(ast.PythonCode(c, self.lineno, self.pos, self.filename).undeclared_identifiers)
+ res += list(ast.PythonCode(c, **self.exception_kwargs).undeclared_identifiers)
return res + list(self.filter_args.undeclared_identifiers.difference(util.Set(filters.DEFAULT_ESCAPES.keys())))
class CallTag(Tag):
__keyword__ = 'call'
def __init__(self, keyword, attributes, **kwargs):
super(CallTag, self).__init__(keyword, attributes, ('args'), ('expr',), ('expr',), **kwargs)
- self.code = ast.PythonCode(attributes['expr'], self.lineno, self.pos, self.filename)
- self.body_decl = ast.FunctionArgs(attributes.get('args', ''), self.lineno, self.pos, self.filename)
+ self.code = ast.PythonCode(attributes['expr'], **self.exception_kwargs)
+ self.body_decl = ast.FunctionArgs(attributes.get('args', ''), **self.exception_kwargs)
def declared_identifiers(self):
return self.code.declared_identifiers.union(self.body_decl.argnames)
def undeclared_identifiers(self):
@@ -281,8 +285,8 @@
__keyword__ = 'page'
def __init__(self, keyword, attributes, **kwargs):
super(PageTag, self).__init__(keyword, attributes, ('cached', 'cache_key', 'cache_timeout', 'cache_type', 'cache_dir', 'cache_url', 'args', 'expression_filter'), (), (), **kwargs)
- self.body_decl = ast.FunctionArgs(attributes.get('args', ''), self.lineno, self.pos, self.filename)
- self.filter_args = ast.ArgumentList(attributes.get('expression_filter', ''), self.lineno, self.pos, self.filename)
+ self.body_decl = ast.FunctionArgs(attributes.get('args', ''), **self.exception_kwargs)
+ self.filter_args = ast.ArgumentList(attributes.get('expression_filter', ''), **self.exception_kwargs)
def declared_identifiers(self):
return self.body_decl.argnames
diff --git a/lib/mako/template.py b/lib/mako/template.py
index 028d11a..8254105 100644
--- a/lib/mako/template.py
+++ b/lib/mako/template.py
@@ -169,15 +169,22 @@
code = property(_get_code)
def _get_source(self):
if self.template_source is not None:
- return self.template_source
+ if self.module._source_encoding and not isinstance(self.template_source, unicode):
+ return self.template_source.decode(self.module._source_encoding)
+ else:
+ return self.template_source
else:
- return file(self.template_filename).read()
+ if self.module._source_encoding:
+ return file(self.template_filename).read().decode(self.module._source_encoding)
+ else:
+ return file(self.template_filename).read()
source = property(_get_source)
def _compile_text(template, text, filename):
identifier = template.module_id
- node = Lexer(text, filename, input_encoding=template.input_encoding, preprocessor=template.preprocessor).parse()
- source = codegen.compile(node, template.uri, filename, default_filters=template.default_filters, buffer_filters=template.buffer_filters, imports=template.imports)
+ lexer = Lexer(text, filename, input_encoding=template.input_encoding, preprocessor=template.preprocessor)
+ node = lexer.parse()
+ source = codegen.compile(node, template.uri, filename, default_filters=template.default_filters, buffer_filters=template.buffer_filters, imports=template.imports, source_encoding=lexer.encoding)
#print source
cid = identifier
module = imp.new_module(cid)
@@ -188,8 +195,9 @@
def _compile_module_file(template, text, filename, outputpath):
identifier = template.module_id
(dest, name) = tempfile.mkstemp()
- node = Lexer(text, filename, input_encoding=template.input_encoding, preprocessor=template.preprocessor).parse()
- source = codegen.compile(node, template.uri, filename, default_filters=template.default_filters, buffer_filters=template.buffer_filters, imports=template.imports)
+ lexer = Lexer(text, filename, input_encoding=template.input_encoding, preprocessor=template.preprocessor)
+ node = lexer.parse()
+ source = codegen.compile(node, template.uri, filename, default_filters=template.default_filters, buffer_filters=template.buffer_filters, imports=template.imports, source_encoding=lexer.encoding)
os.write(dest, source)
os.close(dest)
shutil.move(name, outputpath)
diff --git a/test/ast.py b/test/ast.py
index 4df03fa..f8d604d 100644
--- a/test/ast.py
+++ b/test/ast.py
@@ -3,6 +3,8 @@
from mako import ast, util, exceptions
from compiler import parse
+exception_kwargs = {'source':'', 'lineno':0, 'pos':0, 'filename':''}
+
class AstParseTest(unittest.TestCase):
def setUp(self):
pass
@@ -23,11 +25,11 @@
print "hello world, ", a, b
print "Another expr", c
"""
- parsed = ast.PythonCode(code, 0, 0, None)
+ parsed = ast.PythonCode(code, **exception_kwargs)
assert parsed.declared_identifiers == util.Set(['a','b','c', 'g', 'h', 'i', 'u', 'k', 'j', 'gh', 'lar', 'x'])
assert parsed.undeclared_identifiers == util.Set(['x', 'q', 'foo', 'gah', 'blah'])
- parsed = ast.PythonCode("x + 5 * (y-z)", 0, 0, None)
+ parsed = ast.PythonCode("x + 5 * (y-z)", **exception_kwargs)
assert parsed.undeclared_identifiers == util.Set(['x', 'y', 'z'])
assert parsed.declared_identifiers == util.Set()
@@ -41,7 +43,7 @@
for x in data:
result.append(x+7)
"""
- parsed = ast.PythonCode(code, 0, 0, None)
+ parsed = ast.PythonCode(code, **exception_kwargs)
assert parsed.undeclared_identifiers == util.Set(['get_data'])
assert parsed.declared_identifiers == util.Set(['result', 'data', 'x', 'hoho', 'foobar', 'foo', 'yaya'])
@@ -54,7 +56,7 @@
[z for z in range(1, z)]
(q for q in range (1, q))
"""
- parsed = ast.PythonCode(code, 0, 0, None)
+ parsed = ast.PythonCode(code, **exception_kwargs)
assert parsed.undeclared_identifiers == util.Set(['x', 'y', 'z', 'q'])
def test_locate_identifiers_4(self):
@@ -64,7 +66,7 @@
def mydef(mydefarg):
print "mda is", mydefarg
"""
- parsed = ast.PythonCode(code, 0, 0, None)
+ parsed = ast.PythonCode(code, **exception_kwargs)
assert parsed.undeclared_identifiers == util.Set(['y'])
assert parsed.declared_identifiers == util.Set(['mydef', 'x'])
@@ -75,7 +77,7 @@
except:
print y
"""
- parsed = ast.PythonCode(code, 0, 0, None)
+ parsed = ast.PythonCode(code, **exception_kwargs)
assert parsed.undeclared_identifiers == util.Set(['x', 'y'])
def test_locate_identifiers_6(self):
@@ -83,7 +85,7 @@
def foo():
return bar()
"""
- parsed = ast.PythonCode(code, 0, 0, None)
+ parsed = ast.PythonCode(code, **exception_kwargs)
assert parsed.undeclared_identifiers == util.Set(['bar'])
code = """
@@ -91,7 +93,7 @@
return x, y, z
print x
"""
- parsed = ast.PythonCode(code, 0, 0, None)
+ parsed = ast.PythonCode(code, **exception_kwargs)
assert parsed.undeclared_identifiers == util.Set(['z', 'x'])
assert parsed.declared_identifiers == util.Set(['lala'])
@@ -102,7 +104,7 @@
z = 7
print z
"""
- parsed = ast.PythonCode(code, 0, 0, None)
+ parsed = ast.PythonCode(code, **exception_kwargs)
assert parsed.undeclared_identifiers == util.Set(['z'])
assert parsed.declared_identifiers == util.Set(['lala'])
@@ -110,7 +112,7 @@
code = """
import foo.bar
"""
- parsed = ast.PythonCode(code, 0, 0, None)
+ parsed = ast.PythonCode(code, **exception_kwargs)
assert parsed.declared_identifiers == util.Set(['foo'])
assert parsed.undeclared_identifiers == util.Set()
@@ -121,7 +123,7 @@
def hoho(self):
x = 5
"""
- parsed = ast.PythonCode(code, 0, 0, None)
+ parsed = ast.PythonCode(code, **exception_kwargs)
assert parsed.declared_identifiers == util.Set(['Hi'])
assert parsed.undeclared_identifiers == util.Set()
@@ -131,41 +133,41 @@
import x as bar
"""
try:
- parsed = ast.PythonCode(code, 0, 0, None)
+ parsed = ast.PythonCode(code, **exception_kwargs)
assert False
except exceptions.CompileException, e:
assert str(e).startswith("'import *' is not supported")
def test_python_fragment(self):
- parsed = ast.PythonFragment("for x in foo:", 0, 0, None)
+ parsed = ast.PythonFragment("for x in foo:", **exception_kwargs)
assert parsed.declared_identifiers == util.Set(['x'])
assert parsed.undeclared_identifiers == util.Set(['foo'])
- parsed = ast.PythonFragment("try:", 0, 0, None)
+ parsed = ast.PythonFragment("try:", **exception_kwargs)
- parsed = ast.PythonFragment("except MyException, e:", 0, 0, None)
+ parsed = ast.PythonFragment("except MyException, e:", **exception_kwargs)
assert parsed.declared_identifiers == util.Set(['e'])
assert parsed.undeclared_identifiers == util.Set(['MyException'])
def test_argument_list(self):
- parsed = ast.ArgumentList("3, 5, 'hi', x+5, context.get('lala')", 0, 0, None)
+ parsed = ast.ArgumentList("3, 5, 'hi', x+5, context.get('lala')", **exception_kwargs)
assert parsed.undeclared_identifiers == util.Set(['x', 'context'])
assert [x for x in parsed.args] == ["3", "5", "'hi'", "(x + 5)", "context.get('lala')"]
- parsed = ast.ArgumentList("h", 0, 0, None)
+ parsed = ast.ArgumentList("h", **exception_kwargs)
assert parsed.args == ["h"]
def test_function_decl(self):
"""test getting the arguments from a function"""
code = "def foo(a, b, c=None, d='hi', e=x, f=y+7):pass"
- parsed = ast.FunctionDecl(code, 0, 0, None)
+ parsed = ast.FunctionDecl(code, **exception_kwargs)
assert parsed.funcname=='foo'
assert parsed.argnames==['a', 'b', 'c', 'd', 'e', 'f']
def test_function_decl_2(self):
"""test getting the arguments from a function"""
code = "def foo(a, b, c=None, *args, **kwargs):pass"
- parsed = ast.FunctionDecl(code, 0, 0, None)
+ parsed = ast.FunctionDecl(code, **exception_kwargs)
assert parsed.funcname=='foo'
assert parsed.argnames==['a', 'b', 'c', 'args', 'kwargs']
diff --git a/test/template.py b/test/template.py
index 730f654..a2fc1cc 100644
--- a/test/template.py
+++ b/test/template.py
@@ -3,6 +3,7 @@
from mako.template import Template
from mako.lookup import TemplateLookup
from mako.ext.preprocessors import convert_comments
+from mako import exceptions
import unittest, re, os
from util import flatten_result, result_lines
@@ -12,6 +13,10 @@
os.mkdir('./test_htdocs/subdir')
file('./test_htdocs/unicode.html', 'w').write("""## -*- coding: utf-8 -*-
Alors vous imaginez ma surprise, au lever du jour, quand une drôle de petit voix m’a réveillé. Elle disait: « S’il vous plaît… dessine-moi un mouton! »""")
+file('./test_htdocs/unicode_syntax_error.html', 'w').write("""## -*- coding: utf-8 -*-
+<% print 'Alors vous imaginez ma surprise, au lever du jour, quand une drôle de petit voix m’a réveillé. Elle disait: « S’il vous plaît… dessine-moi un mouton! » %>""")
+file('./test_htdocs/unicode_runtime_error.html', 'w').write("""## -*- coding: utf-8 -*-
+<% print 'Alors vous imaginez ma surprise, au lever du jour, quand une drôle de petit voix m’a réveillé. Elle disait: « S’il vous plaît… dessine-moi un mouton! »' + int(5/0) %>""")
class EncodingTest(unittest.TestCase):
def test_unicode(self):
@@ -243,7 +248,49 @@
""")
assert t.render().strip() == "y is hi"
+class RichTracebackTest(unittest.TestCase):
+ def do_test_traceback(self, utf8, memory, syntax):
+ if memory:
+ if syntax:
+ source = u'## coding: utf-8\n<% print "m’a réveillé. Elle disait: « S’il vous plaît… dessine-moi un mouton! » %>'
+ else:
+ source = u'## coding: utf-8\n<% print u"m’a réveillé. Elle disait: « S’il vous plaît… dessine-moi un mouton! »" + str(5/0) %>'
+ if utf8:
+ source = source.encode('utf-8')
+ else:
+ source = source
+ templateargs = {'text':source}
+ else:
+ if syntax:
+ filename = './test_htdocs/unicode_syntax_error.html'
+ else:
+ filename = './test_htdocs/unicode_runtime_error.html'
+ source = file(filename).read()
+ if not utf8:
+ source = source.decode('utf-8')
+ templateargs = {'filename':filename}
+ try:
+ template = Template(**templateargs)
+ if not syntax:
+ template.render_unicode()
+ assert False
+ except Exception, e:
+ tback = exceptions.RichTraceback()
+ if utf8:
+ assert tback.source == source.decode('utf-8')
+ else:
+ assert tback.source == source
+for utf8 in (True, False):
+ for memory in (True, False):
+ for syntax in (True, False):
+ def do_test(self):
+ self.do_test_traceback(utf8, memory, syntax)
+ name = 'test_%s_%s_%s' % (utf8 and 'utf8' or 'unicode', memory and 'memory' or 'file', syntax and 'syntax' or 'runtime')
+ do_test.__name__ = name
+ setattr(RichTracebackTest, name, do_test)
+
+
class ModuleDirTest(unittest.TestCase):
def test_basic(self):
file('./test_htdocs/modtest.html', 'w').write("""this is a test""")
