- ensure lru threading test doesn't run
- Source code escaping has been simplified.
In particular, module source files are now
generated with the Python "magic encoding
comment", and source code is passed through
mostly unescaped, except for that code which
is regenerated from parsed Python source.
This fixes usage of unicode in
<%namespace:defname> tags. [ticket:99]
diff --git a/CHANGES b/CHANGES
index 66f47ed..f048aed 100644
--- a/CHANGES
+++ b/CHANGES
@@ -1,9 +1,16 @@
0.3
-- Python 2.3 support is dropped (who can resist
- @decorators) [ticket:123]
+- Python 2.3 support is dropped. [ticket:123]
- Unit tests now run with nose. [ticket:127]
+- Source code escaping has been simplified.
+ In particular, module source files are now
+ generated with the Python "magic encoding
+ comment", and source code is passed through
+ mostly unescaped, except for that code which
+ is regenerated from parsed Python source.
+ This fixes usage of unicode in
+ <%namespace:defname> tags. [ticket:99]
0.2.6
diff --git a/mako/__init__.py b/mako/__init__.py
index 6a30418..2b7e0cf 100644
--- a/mako/__init__.py
+++ b/mako/__init__.py
@@ -5,5 +5,5 @@
# the MIT License: http://www.opensource.org/licenses/mit-license.php
-__version__ = '0.2.5'
+__version__ = '0.3.0'
diff --git a/mako/codegen.py b/mako/codegen.py
index b3074f0..19fe18e 100644
--- a/mako/codegen.py
+++ b/mako/codegen.py
@@ -20,12 +20,12 @@
buffer_filters=None,
imports=None,
source_encoding=None,
- generate_unicode=True):
+ generate_magic_comment=True):
"""Generate module source code given a parsetree node,
uri, and optional source filename"""
- buf = util.FastEncodingBuffer(unicode=generate_unicode)
+ buf = util.FastEncodingBuffer()
printer = PythonPrinter(buf)
_GenerateRenderMethod(printer,
@@ -35,7 +35,7 @@
buffer_filters,
imports,
source_encoding,
- generate_unicode),
+ generate_magic_comment),
node)
return buf.getvalue()
@@ -47,14 +47,14 @@
buffer_filters,
imports,
source_encoding,
- generate_unicode):
+ generate_magic_comment):
self.uri = uri
self.filename = filename
self.default_filters = default_filters
self.buffer_filters = buffer_filters
self.imports = imports
self.source_encoding = source_encoding
- self.generate_unicode = generate_unicode
+ self.generate_magic_comment = generate_magic_comment
class _GenerateRenderMethod(object):
"""A template visitor object which generates the
@@ -146,7 +146,7 @@
module_identifiers.declared = module_ident
# module-level names, python code
- if not self.compiler.generate_unicode and \
+ if self.compiler.generate_magic_comment and \
self.compiler.source_encoding:
self.printer.writeline("# -*- encoding:%s -*-" %
self.compiler.source_encoding)
diff --git a/mako/lexer.py b/mako/lexer.py
index 65be795..caf295b 100644
--- a/mako/lexer.py
+++ b/mako/lexer.py
@@ -128,15 +128,10 @@
(node.keyword, self.control_line[-1].keyword),
**self.exception_kwargs)
- def escape_code(self, text):
- if not self.disable_unicode and self.encoding:
- return text.encode('ascii', 'backslashreplace')
- else:
- return text
-
def parse(self):
for preproc in self.preprocessor:
self.text = preproc(self.text)
+
if not isinstance(self.text, unicode) and self.text.startswith(codecs.BOM_UTF8):
self.text = self.text[len(codecs.BOM_UTF8):]
parsed_encoding = 'utf-8'
@@ -149,6 +144,7 @@
0, 0, self.filename)
else:
parsed_encoding = self.match_encoding()
+
if parsed_encoding:
self.encoding = parsed_encoding
@@ -242,7 +238,7 @@
key, val1, val2 = att
text = val1 or val2
text = text.replace('\r\n', '\n')
- attributes[key] = self.escape_code(text)
+ attributes[key] = text
self.append_node(parsetree.Tag, keyword, attributes)
if isend:
self.tag.pop()
@@ -326,7 +322,7 @@
text = adjust_whitespace(text) + "\n"
self.append_node(
parsetree.Code,
- self.escape_code(text),
+ text,
match.group(1)=='!', lineno=line, pos=pos)
return True
else:
@@ -344,7 +340,7 @@
text = text.replace('\r\n', '\n')
self.append_node(
parsetree.Expression,
- self.escape_code(text), escapes.strip(),
+ text, escapes.strip(),
lineno=line, pos=pos)
return True
else:
@@ -376,7 +372,7 @@
"Keyword '%s' doesn't match keyword '%s'" %
(text, self.control_line[-1].keyword),
**self.exception_kwargs)
- self.append_node(parsetree.ControlLine, keyword, isend, self.escape_code(text))
+ self.append_node(parsetree.ControlLine, keyword, isend, text)
else:
self.append_node(parsetree.Comment, text)
return True
diff --git a/mako/pyparser.py b/mako/pyparser.py
index c79692c..34b2a6a 100644
--- a/mako/pyparser.py
+++ b/mako/pyparser.py
@@ -28,10 +28,14 @@
def parse(code, mode='exec', **exception_kwargs):
"""Parse an expression into AST"""
+
+
try:
if _ast:
return _ast_util.parse(code, '<unknown>', mode)
else:
+ if isinstance(code, unicode):
+ code = code.encode('ascii', 'backslashreplace')
return compiler_parse(code, mode)
except Exception, e:
raise exceptions.SyntaxException("(%s) %s (%s)" % (e.__class__.__name__, str(e), repr(code[0:50])), **exception_kwargs)
diff --git a/mako/runtime.py b/mako/runtime.py
index a475b71..583b79e 100644
--- a/mako/runtime.py
+++ b/mako/runtime.py
@@ -361,7 +361,10 @@
if as_unicode:
buf = util.FastEncodingBuffer(unicode=True)
elif template.output_encoding:
- buf = util.FastEncodingBuffer(unicode=as_unicode, encoding=template.output_encoding, errors=template.encoding_errors)
+ buf = util.FastEncodingBuffer(
+ unicode=as_unicode,
+ encoding=template.output_encoding,
+ errors=template.encoding_errors)
else:
buf = util.StringIO()
context = Context(buf, **data)
diff --git a/mako/template.py b/mako/template.py
index 4bf01e0..87f6898 100644
--- a/mako/template.py
+++ b/mako/template.py
@@ -352,7 +352,7 @@
buffer_filters=template.buffer_filters,
imports=template.imports,
source_encoding=lexer.encoding,
- generate_unicode=not template.disable_unicode)
+ generate_magic_comment=template.disable_unicode)
cid = identifier
if isinstance(cid, unicode):
@@ -378,8 +378,12 @@
buffer_filters=template.buffer_filters,
imports=template.imports,
source_encoding=lexer.encoding,
- generate_unicode=not template.disable_unicode)
+ generate_magic_comment=True)
(dest, name) = tempfile.mkstemp()
+
+ if isinstance(source, unicode):
+ source = source.encode(lexer.encoding or 'ascii')
+
os.write(dest, source)
os.close(dest)
shutil.move(name, outputpath)
diff --git a/mako/util.py b/mako/util.py
index 472535d..6119fa0 100644
--- a/mako/util.py
+++ b/mako/util.py
@@ -72,7 +72,8 @@
return x
class FastEncodingBuffer(object):
- """a very rudimentary buffer that is faster than StringIO, but doesnt crash on unicode data like cStringIO."""
+ """a very rudimentary buffer that is faster than StringIO,
+ but doesnt crash on unicode data like cStringIO."""
def __init__(self, encoding=None, errors='strict', unicode=False):
self.data = []
diff --git a/setup.py b/setup.py
index fe03b70..caed701 100644
--- a/setup.py
+++ b/setup.py
@@ -25,7 +25,7 @@
""",
classifiers=[
- 'Development Status :: 4 - Beta',
+ "Development Status :: 5 - Production/Stable",
'Environment :: Web Environment',
'Intended Audience :: Developers',
'Programming Language :: Python',
diff --git a/test/test_lru.py b/test/test_lru.py
index d75b47a..e2b5c15 100644
--- a/test/test_lru.py
+++ b/test/test_lru.py
@@ -41,7 +41,7 @@
for id in (25, 24, 23, 14, 12, 19, 18, 17, 16, 15):
self.assert_(l.has_key(id))
- def disabled_test_threaded(self):
+ def _disabled_test_threaded(self):
size = 100
threshold = .5
all_elems = 2000
@@ -76,6 +76,7 @@
cache[id] = e
time.sleep(random.random() / 1000)
+
for x in range(0,20):
thread.start_new_thread(request_elem, ())
@@ -100,7 +101,9 @@
total_avg = average_regets_in_range(0, 2000)
# hotzone should be way above the others
- print "total fetches", total[0], "hotzone", hotzone_avg, "control", control_avg, "total", total_avg
+ print "total fetches", total[0], "hotzone", \
+ hotzone_avg, "control", \
+ control_avg, "total", total_avg
assert hotzone_avg > total_avg * 5 > control_avg * 5
diff --git a/test/test_template.py b/test/test_template.py
index 8b6b7d2..f06ab5b 100644
--- a/test/test_template.py
+++ b/test/test_template.py
@@ -142,6 +142,29 @@
filters=lambda s:s.strip(),
)
+ def test_unicode_literal_in_tag(self):
+ self._do_file_test(
+ "unicode_arguments.html",
+ [
+ u'x is: drôle de petit voix m’a réveillé',
+ u'x is: drôle de petit voix m’a réveillé',
+ u'x is: drôle de petit voix m’a réveillé',
+ u'x is: drôle de petit voix m’a réveillé',
+ ],
+ filters=result_lines
+ )
+
+ self._do_memory_test(
+ file(self._file_path("unicode_arguments.html")).read(),
+ [
+ u'x is: drôle de petit voix m’a réveillé',
+ u'x is: drôle de petit voix m’a réveillé',
+ u'x is: drôle de petit voix m’a réveillé',
+ u'x is: drôle de petit voix m’a réveillé',
+ ],
+ filters=result_lines
+ )
+
def test_unicode_literal_in_def(self):
self._do_memory_test(
u"""## -*- coding: utf-8 -*-
