- [feature] Added lexer_cls argument to Template,
  TemplateLookup, allows alternate Lexer classes
  to be used.

CHANGES

diff --git a/CHANGES b/CHANGES
index 6f0c417..8626e24 100644
--- a/CHANGES
+++ b/CHANGES
@@ -2,6 +2,10 @@
 - [feature] Code has been reworked to support Python 2.4->
   Python 3.xx in place.  2to3 no longer needed.
 
+- [feature] Added lexer_cls argument to Template,
+  TemplateLookup, allows alternate Lexer classes
+  to be used.
+
 - [feature] Added future_imports parameter to Template
   and TemplateLookup, renders the __future__ header
   with desired capabilities at the top of the generated

mako/codegen.py

diff --git a/mako/codegen.py b/mako/codegen.py
index f730127..fcf0fb7 100644
--- a/mako/codegen.py
+++ b/mako/codegen.py
@@ -103,7 +103,6 @@
         self.compiler = compiler
         self.node = node
         self.identifier_stack = [None]
-
         self.in_def = isinstance(node, (parsetree.DefTag, parsetree.BlockTag))
 
         if self.in_def:
@@ -159,7 +158,6 @@
         inherit = []
         namespaces = {}
         module_code = []
-        encoding = [None]
 
         self.compiler.pagetag = None
 

mako/lookup.py

diff --git a/mako/lookup.py b/mako/lookup.py
index a67e38e..183b93b 100644
--- a/mako/lookup.py
+++ b/mako/lookup.py
@@ -172,7 +172,8 @@
                         future_imports=None,
                         enable_loop=True,
                         input_encoding=None,
-                        preprocessor=None):
+                        preprocessor=None,
+                        lexer_cls=None):
 
         self.directories = [posixpath.normpath(d) for d in
                             util.to_list(directories, ())
@@ -211,7 +212,9 @@
             'imports':imports,
             'future_imports':future_imports,
             'enable_loop':enable_loop,
-            'preprocessor':preprocessor}
+            'preprocessor':preprocessor,
+            'lexer_cls':lexer_cls
+        }
 
         if collection_size == -1:
             self._collection = {}

mako/template.py

diff --git a/mako/template.py b/mako/template.py
index 9a513e3..a246f7e 100644
--- a/mako/template.py
+++ b/mako/template.py
@@ -188,6 +188,12 @@
      result of the callable will be used as the template source
      code.
 
+    :param lexer_cls: A :class:`.Lexer` class used to parse
+     the template.   The :class:`.Lexer` class is used by
+     default.
+
+     .. versionadded:: 0.7.4
+
     :param strict_undefined: Replaces the automatic usage of
      ``UNDEFINED`` for any undeclared variables not located in
      the :class:`.Context` with an immediate raise of
@@ -206,6 +212,8 @@
 
     """
 
+    lexer_cls = Lexer
+
     def __init__(self,
                     text=None,
                     filename=None,
@@ -233,7 +241,8 @@
                     imports=None,
                     future_imports=None,
                     enable_loop=True,
-                    preprocessor=None):
+                    preprocessor=None,
+                    lexer_cls=None):
         if uri:
             self.module_id = re.sub(r'\W', "_", uri)
             self.uri = uri
@@ -286,6 +295,9 @@
         self.future_imports = future_imports
         self.preprocessor = preprocessor
 
+        if lexer_cls is not None:
+            self.lexer_cls = lexer_cls
+
         # if plain text, compile code in memory only
         if text is not None:
             (code, module) = _compile_text(self, text, filename)
@@ -608,11 +620,11 @@
                 return data
 
 def _compile(template, text, filename, generate_magic_comment):
-    lexer = Lexer(text,
-                    filename,
-                    disable_unicode=template.disable_unicode,
-                    input_encoding=template.input_encoding,
-                    preprocessor=template.preprocessor)
+    lexer = template.lexer_cls(text,
+                           filename,
+                           disable_unicode=template.disable_unicode,
+                           input_encoding=template.input_encoding,
+                           preprocessor=template.preprocessor)
     node = lexer.parse()
     source = codegen.compile(node,
                             template.uri,

test/test_template.py

diff --git a/test/test_template.py b/test/test_template.py
index 28d397d..25b2fae 100644
--- a/test/test_template.py
+++ b/test/test_template.py
@@ -3,12 +3,10 @@
 from mako.template import Template, ModuleTemplate
 from mako.lookup import TemplateLookup
 from mako.ext.preprocessors import convert_comments
-from mako import exceptions, util, runtime
+from mako import exceptions, runtime
 from mako import compat
-import re
 import os
 from test.util import flatten_result, result_lines
-import codecs
 from mako.compat import u
 from test import TemplateTest, eq_, template_base, module_base, \
     requires_python_26_or_greater, assert_raises, assert_raises_message, \
@@ -1239,6 +1237,38 @@
 
         assert flatten_result(t.render()) == "im a template - # not a comment - ## not a comment"
 
+class LexerTest(TemplateTest):
+    def _fixture(self):
+        from mako.parsetree import TemplateNode, Text
+        class MyLexer(object):
+            encoding = 'ascii'
+            def __init__(self, *arg, **kw):
+                pass
+
+            def parse(self):
+                t = TemplateNode("foo")
+                t.nodes.append(
+                    Text("hello world", source="foo", lineno=0,
+                            pos=0, filename=None)
+                )
+                return t
+        return MyLexer
+
+    def _test_custom_lexer(self, template):
+        eq_(
+            result_lines(template.render()),
+            ["hello world"]
+        )
+
+    def test_via_template(self):
+        t = Template("foo", lexer_cls=self._fixture())
+        self._test_custom_lexer(t)
+
+    def test_via_lookup(self):
+        tl = TemplateLookup(lexer_cls=self._fixture())
+        tl.put_string("foo", "foo")
+        t = tl.get_template("foo")
+        self._test_custom_lexer(t)
 
 class FuturesTest(TemplateTest):