- added "preprocessor" argument to Template, TemplateLookup - is a single
callable or list of callables which will be applied to the template text
before lexing. given the text as an argument, returns the new text.
- added mako.ext.preprocessors package, contains one preprocessor so far:
'convert_comments', which will convert single # comments to the new ##
format
diff --git a/CHANGES b/CHANGES
index 711b2a9..f160bc5 100644
--- a/CHANGES
+++ b/CHANGES
@@ -9,7 +9,13 @@
- UNDEFINED evaluates to False
- improvement to scoping of "caller" variable when using <%call> tag
- added lexer error for unclosed control-line (%) line
-
+- added "preprocessor" argument to Template, TemplateLookup - is a single
+ callable or list of callables which will be applied to the template text
+ before lexing. given the text as an argument, returns the new text.
+- added mako.ext.preprocessors package, contains one preprocessor so far:
+ 'convert_comments', which will convert single # comments to the new ##
+ format
+
0.1.2
- fix to parsing of code/expression blocks to insure that non-ascii
characters, combined with a template that indicates a non-standard
diff --git a/lib/mako/ext/preprocessors.py b/lib/mako/ext/preprocessors.py
new file mode 100644
index 0000000..601ac0d
--- /dev/null
+++ b/lib/mako/ext/preprocessors.py
@@ -0,0 +1,20 @@
+"""preprocessing functions, used with the 'preprocessor' argument on Template, TemplateLookup"""
+
+import re
+
+def convert_comments(text):
+ """preprocess old style comments.
+
+ example:
+
+ from mako.ext.preprocessors import convert_comments
+ t = Template(..., preprocessor=preprocess_comments)"""
+ return re.sub(r'(?<=\n)\s*#[^#]', "##", text)
+
+# TODO
+def create_tag(callable):
+ """given a callable, extract the *args and **kwargs, and produce a preprocessor
+ that will parse for <%<funcname> <args>> and convert to an appropriate <%call> statement.
+
+ this allows any custom tag to be created which looks like a pure Mako-style tag."""
+ raise NotImplementedError("Future functionality....")
\ No newline at end of file
diff --git a/lib/mako/lexer.py b/lib/mako/lexer.py
index 0d9081d..9e0905f 100644
--- a/lib/mako/lexer.py
+++ b/lib/mako/lexer.py
@@ -11,7 +11,7 @@
from mako.pygen import adjust_whitespace
class Lexer(object):
- def __init__(self, text, filename=None, input_encoding=None):
+ def __init__(self, text, filename=None, input_encoding=None, preprocessor=None):
self.text = text
self.filename = filename
self.template = parsetree.TemplateNode(self.filename)
@@ -22,6 +22,12 @@
self.tag = []
self.control_line = []
self.encoding = input_encoding
+ if preprocessor is None:
+ self.preprocessor = []
+ elif not hasattr(preprocessor, '__iter__'):
+ self.preprocessor = [preprocessor]
+ else:
+ self.preprocessor = preprocessor
def match(self, regexp, flags=None):
"""match the given regular expression string and flags to the current text position.
@@ -95,6 +101,8 @@
return text
def parse(self):
+ for preproc in self.preprocessor:
+ self.text = preproc(self.text)
parsed_encoding = self.match_encoding()
if parsed_encoding:
self.encoding = parsed_encoding
diff --git a/lib/mako/lookup.py b/lib/mako/lookup.py
index 0ec01e1..b975ac6 100644
--- a/lib/mako/lookup.py
+++ b/lib/mako/lookup.py
@@ -37,7 +37,7 @@
return uri
class TemplateLookup(TemplateCollection):
- def __init__(self, directories=None, module_directory=None, filesystem_checks=True, collection_size=-1, format_exceptions=False, error_handler=None, output_encoding=None, cache_type=None, cache_dir=None, modulename_callable=None, default_filters=['unicode'], imports=None, input_encoding=None):
+ def __init__(self, directories=None, module_directory=None, filesystem_checks=True, collection_size=-1, format_exceptions=False, error_handler=None, output_encoding=None, cache_type=None, cache_dir=None, modulename_callable=None, default_filters=['unicode'], imports=None, input_encoding=None, preprocessor=None):
if isinstance(directories, basestring):
directories = [directories]
self.directories = [posixpath.normpath(d) for d in directories or []]
@@ -45,7 +45,7 @@
self.modulename_callable = modulename_callable
self.filesystem_checks = filesystem_checks
self.collection_size = collection_size
- self.template_args = {'format_exceptions':format_exceptions, 'error_handler':error_handler, 'output_encoding':output_encoding, 'input_encoding':input_encoding, 'module_directory':module_directory, 'cache_type':cache_type, 'cache_dir':cache_dir or module_directory, 'default_filters':default_filters, 'imports':imports}
+ self.template_args = {'format_exceptions':format_exceptions, 'error_handler':error_handler, 'output_encoding':output_encoding, 'input_encoding':input_encoding, 'module_directory':module_directory, 'cache_type':cache_type, 'cache_dir':cache_dir or module_directory, 'default_filters':default_filters, 'imports':imports, 'preprocessor':preprocessor}
if collection_size == -1:
self.__collection = {}
self._uri_cache = {}
diff --git a/lib/mako/template.py b/lib/mako/template.py
index de6ba1f..7b8608a 100644
--- a/lib/mako/template.py
+++ b/lib/mako/template.py
@@ -16,7 +16,7 @@
class Template(object):
"""a compiled template"""
- def __init__(self, text=None, filename=None, uri=None, format_exceptions=False, error_handler=None, lookup=None, output_encoding=None, module_directory=None, cache_type=None, cache_dir=None, module_filename=None, input_encoding=None, default_filters=['unicode'], imports=None):
+ def __init__(self, text=None, filename=None, uri=None, format_exceptions=False, error_handler=None, lookup=None, output_encoding=None, module_directory=None, cache_type=None, cache_dir=None, module_filename=None, input_encoding=None, default_filters=['unicode'], imports=None, preprocessor=None):
"""construct a new Template instance using either literal template text, or a previously loaded template module
text - textual template source, or None if a module is to be provided
@@ -41,6 +41,7 @@
self.default_filters = default_filters
self.input_encoding = input_encoding
self.imports = imports
+ self.preprocessor = preprocessor
# if plain text, compile code in memory only
if text is not None:
@@ -171,7 +172,7 @@
def _compile_text(template, text, filename):
identifier = template.module_id
- node = Lexer(text, filename, input_encoding=template.input_encoding).parse()
+ node = Lexer(text, filename, input_encoding=template.input_encoding, preprocessor=template.preprocessor).parse()
source = codegen.compile(node, template.uri, filename, default_filters=template.default_filters, imports=template.imports)
cid = identifier
module = imp.new_module(cid)
@@ -182,7 +183,7 @@
def _compile_module_file(template, text, filename, outputpath):
identifier = template.module_id
(dest, name) = tempfile.mkstemp()
- node = Lexer(text, filename, input_encoding=template.input_encoding).parse()
+ node = Lexer(text, filename, input_encoding=template.input_encoding, preprocessor=template.preprocessor).parse()
source = codegen.compile(node, template.uri, filename, default_filters=template.default_filters, imports=template.imports)
os.write(dest, source)
os.close(dest)
diff --git a/test/lexer.py b/test/lexer.py
index 8c76202..b12a756 100644
--- a/test/lexer.py
+++ b/test/lexer.py
@@ -4,6 +4,7 @@
from mako import exceptions
from util import flatten_result, result_lines
from mako.template import Template
+import re
class LexerTest(unittest.TestCase):
def test_text_and_tag(self):
@@ -376,6 +377,17 @@
"""
nodes = Lexer(template).parse()
assert repr(nodes) == r"""TemplateNode({}, [Text(u'\n<style>\n #someselector\n # other non comment stuff\n</style>\n', (1, 1)), Comment(u'a comment', (6, 1)), Text(u'\n# also not a comment\n\n', (7, 1)), Comment(u'this is a comment', (10, 1)), Text(u' \nthis is ## not a comment\n\n', (11, 1)), Comment(u' multiline\ncomment\n', (14, 1)), Text(u'\n\nhi\n', (16, 8))])"""
+
+ def test_preprocess(self):
+ def preproc(text):
+ return re.sub(r'(?<=\n)\s*#[^#]', "##", text)
+ template = """
+ hi
+ # old style comment
+# another comment
+"""
+ nodes = Lexer(template, preprocessor=preproc).parse()
+ assert repr(nodes) == r"""TemplateNode({}, [Text(u'\n hi\n', (1, 1)), Comment(u'old style comment', (3, 1)), Comment(u'another comment', (4, 1))])"""
if __name__ == '__main__':
unittest.main()
diff --git a/test/template.py b/test/template.py
index 2611e39..871b34f 100644
--- a/test/template.py
+++ b/test/template.py
@@ -2,6 +2,7 @@
from mako.template import Template
from mako.lookup import TemplateLookup
+from mako.ext.preprocessors import convert_comments
import unittest, re, os
from util import flatten_result, result_lines
@@ -196,7 +197,20 @@
t2 = lookup.get_template('/subdir/modtest.html')
assert t.module.__file__ == 'test_htdocs/foo/modtest.html.py'
assert t2.module.__file__ == 'test_htdocs/subdir/foo/modtest.html.py'
-
+
+class PreprocessTest(unittest.TestCase):
+ def test_old_comments(self):
+ t = Template("""
+ im a template
+# old style comment
+ # more old style comment
+
+ ## new style comment
+ - # not a comment
+ - ## not a comment
+""", preprocessor=convert_comments)
+
+ assert flatten_result(t.render()) == "im a template - # not a comment - ## not a comment"
if __name__ == '__main__':
unittest.main()
