| """ |
| A small templating language |
| |
| This implements a small templating language for use internally in |
| Paste and Paste Script. This language implements if/elif/else, |
| for/continue/break, expressions, and blocks of Python code. The |
| syntax is:: |
| |
| {{any expression (function calls etc)}} |
| {{any expression | filter}} |
| {{for x in y}}...{{endfor}} |
| {{if x}}x{{elif y}}y{{else}}z{{endif}} |
| {{py:x=1}} |
| {{py: |
| def foo(bar): |
| return 'baz' |
| }} |
| {{default var = default_value}} |
| {{# comment}} |
| |
| You use this with the ``Template`` class or the ``sub`` shortcut. |
| The ``Template`` class takes the template string and the name of |
| the template (for errors) and a default namespace. Then (like |
| ``string.Template``) you can call the ``tmpl.substitute(**kw)`` |
| method to make a substitution (or ``tmpl.substitute(a_dict)``). |
| |
| ``sub(content, **kw)`` substitutes the template immediately. You |
| can use ``__name='tmpl.html'`` to set the name of the template. |
| |
| If there are syntax errors ``TemplateError`` will be raised. |
| """ |
| |
| import re |
| import six |
| import sys |
| import cgi |
| from six.moves.urllib.parse import quote |
| from paste.util.looper import looper |
| |
| __all__ = ['TemplateError', 'Template', 'sub', 'HTMLTemplate', |
| 'sub_html', 'html', 'bunch'] |
| |
| token_re = re.compile(r'\{\{|\}\}') |
| in_re = re.compile(r'\s+in\s+') |
| var_re = re.compile(r'^[a-z_][a-z0-9_]*$', re.I) |
| |
| class TemplateError(Exception): |
| """Exception raised while parsing a template |
| """ |
| |
| def __init__(self, message, position, name=None): |
| self.message = message |
| self.position = position |
| self.name = name |
| |
| def __str__(self): |
| msg = '%s at line %s column %s' % ( |
| self.message, self.position[0], self.position[1]) |
| if self.name: |
| msg += ' in %s' % self.name |
| return msg |
| |
| class _TemplateContinue(Exception): |
| pass |
| |
| class _TemplateBreak(Exception): |
| pass |
| |
| class Template(object): |
| |
| default_namespace = { |
| 'start_braces': '{{', |
| 'end_braces': '}}', |
| 'looper': looper, |
| } |
| |
| default_encoding = 'utf8' |
| |
| def __init__(self, content, name=None, namespace=None): |
| self.content = content |
| self._unicode = isinstance(content, six.text_type) |
| self.name = name |
| |
| if not self._unicode: |
| content = content.decode(self.default_encoding) |
| self._unicode = True |
| |
| self._parsed = parse(content, name=name) |
| if namespace is None: |
| namespace = {} |
| self.namespace = namespace |
| |
| def from_filename(cls, filename, namespace=None, encoding=None): |
| f = open(filename, 'rb') |
| c = f.read() |
| f.close() |
| if encoding: |
| c = c.decode(encoding) |
| return cls(content=c, name=filename, namespace=namespace) |
| |
| from_filename = classmethod(from_filename) |
| |
| def __repr__(self): |
| return '<%s %s name=%r>' % ( |
| self.__class__.__name__, |
| hex(id(self))[2:], self.name) |
| |
| def substitute(self, *args, **kw): |
| if args: |
| if kw: |
| raise TypeError( |
| "You can only give positional *or* keyword arguments") |
| if len(args) > 1: |
| raise TypeError( |
| "You can only give on positional argument") |
| kw = args[0] |
| ns = self.default_namespace.copy() |
| ns.update(self.namespace) |
| ns.update(kw) |
| result = self._interpret(ns) |
| return result |
| |
| def _interpret(self, ns): |
| __traceback_hide__ = True |
| parts = [] |
| self._interpret_codes(self._parsed, ns, out=parts) |
| return ''.join(parts) |
| |
| def _interpret_codes(self, codes, ns, out): |
| __traceback_hide__ = True |
| for item in codes: |
| if isinstance(item, six.string_types): |
| out.append(item) |
| else: |
| self._interpret_code(item, ns, out) |
| |
| def _interpret_code(self, code, ns, out): |
| __traceback_hide__ = True |
| name, pos = code[0], code[1] |
| if name == 'py': |
| self._exec(code[2], ns, pos) |
| elif name == 'continue': |
| raise _TemplateContinue() |
| elif name == 'break': |
| raise _TemplateBreak() |
| elif name == 'for': |
| vars, expr, content = code[2], code[3], code[4] |
| expr = self._eval(expr, ns, pos) |
| self._interpret_for(vars, expr, content, ns, out) |
| elif name == 'cond': |
| parts = code[2:] |
| self._interpret_if(parts, ns, out) |
| elif name == 'expr': |
| parts = code[2].split('|') |
| base = self._eval(parts[0], ns, pos) |
| for part in parts[1:]: |
| func = self._eval(part, ns, pos) |
| base = func(base) |
| out.append(self._repr(base, pos)) |
| elif name == 'default': |
| var, expr = code[2], code[3] |
| if var not in ns: |
| result = self._eval(expr, ns, pos) |
| ns[var] = result |
| elif name == 'comment': |
| return |
| else: |
| assert 0, "Unknown code: %r" % name |
| |
| def _interpret_for(self, vars, expr, content, ns, out): |
| __traceback_hide__ = True |
| for item in expr: |
| if len(vars) == 1: |
| ns[vars[0]] = item |
| else: |
| if len(vars) != len(item): |
| raise ValueError( |
| 'Need %i items to unpack (got %i items)' |
| % (len(vars), len(item))) |
| for name, value in zip(vars, item): |
| ns[name] = value |
| try: |
| self._interpret_codes(content, ns, out) |
| except _TemplateContinue: |
| continue |
| except _TemplateBreak: |
| break |
| |
| def _interpret_if(self, parts, ns, out): |
| __traceback_hide__ = True |
| # @@: if/else/else gets through |
| for part in parts: |
| assert not isinstance(part, six.string_types) |
| name, pos = part[0], part[1] |
| if name == 'else': |
| result = True |
| else: |
| result = self._eval(part[2], ns, pos) |
| if result: |
| self._interpret_codes(part[3], ns, out) |
| break |
| |
| def _eval(self, code, ns, pos): |
| __traceback_hide__ = True |
| try: |
| value = eval(code, ns) |
| return value |
| except: |
| exc_info = sys.exc_info() |
| e = exc_info[1] |
| if getattr(e, 'args'): |
| arg0 = e.args[0] |
| else: |
| arg0 = str(e) |
| e.args = (self._add_line_info(arg0, pos),) |
| six.reraise(exc_info[0], e, exc_info[2]) |
| |
| def _exec(self, code, ns, pos): |
| __traceback_hide__ = True |
| try: |
| six.exec_(code, ns) |
| except: |
| exc_info = sys.exc_info() |
| e = exc_info[1] |
| e.args = (self._add_line_info(e.args[0], pos),) |
| six.reraise(exc_info[0], e, exc_info[2]) |
| |
| def _repr(self, value, pos): |
| __traceback_hide__ = True |
| try: |
| if value is None: |
| return '' |
| if self._unicode: |
| try: |
| value = six.text_type(value) |
| except UnicodeDecodeError: |
| value = str(value) |
| else: |
| value = str(value) |
| except: |
| exc_info = sys.exc_info() |
| e = exc_info[1] |
| e.args = (self._add_line_info(e.args[0], pos),) |
| six.reraise(exc_info[0], e, exc_info[2]) |
| else: |
| if self._unicode and isinstance(value, six.binary_type): |
| if not self.default_encoding: |
| raise UnicodeDecodeError( |
| 'Cannot decode str value %r into unicode ' |
| '(no default_encoding provided)' % value) |
| value = value.decode(self.default_encoding) |
| elif not self._unicode and isinstance(value, six.text_type): |
| if not self.default_encoding: |
| raise UnicodeEncodeError( |
| 'Cannot encode unicode value %r into str ' |
| '(no default_encoding provided)' % value) |
| value = value.encode(self.default_encoding) |
| return value |
| |
| |
| def _add_line_info(self, msg, pos): |
| msg = "%s at line %s column %s" % ( |
| msg, pos[0], pos[1]) |
| if self.name: |
| msg += " in file %s" % self.name |
| return msg |
| |
| def sub(content, **kw): |
| name = kw.get('__name') |
| tmpl = Template(content, name=name) |
| return tmpl.substitute(kw) |
| |
| def paste_script_template_renderer(content, vars, filename=None): |
| tmpl = Template(content, name=filename) |
| return tmpl.substitute(vars) |
| |
| class bunch(dict): |
| |
| def __init__(self, **kw): |
| for name, value in kw.items(): |
| setattr(self, name, value) |
| |
| def __setattr__(self, name, value): |
| self[name] = value |
| |
| def __getattr__(self, name): |
| try: |
| return self[name] |
| except KeyError: |
| raise AttributeError(name) |
| |
| def __getitem__(self, key): |
| if 'default' in self: |
| try: |
| return dict.__getitem__(self, key) |
| except KeyError: |
| return dict.__getitem__(self, 'default') |
| else: |
| return dict.__getitem__(self, key) |
| |
| def __repr__(self): |
| items = [ |
| (k, v) for k, v in self.items()] |
| items.sort() |
| return '<%s %s>' % ( |
| self.__class__.__name__, |
| ' '.join(['%s=%r' % (k, v) for k, v in items])) |
| |
| ############################################################ |
| ## HTML Templating |
| ############################################################ |
| |
| class html(object): |
| def __init__(self, value): |
| self.value = value |
| def __str__(self): |
| return self.value |
| def __repr__(self): |
| return '<%s %r>' % ( |
| self.__class__.__name__, self.value) |
| |
| def html_quote(value): |
| if value is None: |
| return '' |
| if not isinstance(value, six.string_types): |
| if hasattr(value, '__unicode__'): |
| value = unicode(value) |
| else: |
| value = str(value) |
| value = cgi.escape(value, 1) |
| if isinstance(value, unicode): |
| value = value.encode('ascii', 'xmlcharrefreplace') |
| return value |
| |
| def url(v): |
| if not isinstance(v, six.string_types): |
| if hasattr(v, '__unicode__'): |
| v = unicode(v) |
| else: |
| v = str(v) |
| if isinstance(v, unicode): |
| v = v.encode('utf8') |
| return quote(v) |
| |
| def attr(**kw): |
| kw = kw.items() |
| kw.sort() |
| parts = [] |
| for name, value in kw: |
| if value is None: |
| continue |
| if name.endswith('_'): |
| name = name[:-1] |
| parts.append('%s="%s"' % (html_quote(name), html_quote(value))) |
| return html(' '.join(parts)) |
| |
| class HTMLTemplate(Template): |
| |
| default_namespace = Template.default_namespace.copy() |
| default_namespace.update(dict( |
| html=html, |
| attr=attr, |
| url=url, |
| )) |
| |
| def _repr(self, value, pos): |
| plain = Template._repr(self, value, pos) |
| if isinstance(value, html): |
| return plain |
| else: |
| return html_quote(plain) |
| |
| def sub_html(content, **kw): |
| name = kw.get('__name') |
| tmpl = HTMLTemplate(content, name=name) |
| return tmpl.substitute(kw) |
| |
| |
| ############################################################ |
| ## Lexing and Parsing |
| ############################################################ |
| |
| def lex(s, name=None, trim_whitespace=True): |
| """ |
| Lex a string into chunks: |
| |
| >>> lex('hey') |
| ['hey'] |
| >>> lex('hey {{you}}') |
| ['hey ', ('you', (1, 7))] |
| >>> lex('hey {{') |
| Traceback (most recent call last): |
| ... |
| TemplateError: No }} to finish last expression at line 1 column 7 |
| >>> lex('hey }}') |
| Traceback (most recent call last): |
| ... |
| TemplateError: }} outside expression at line 1 column 7 |
| >>> lex('hey {{ {{') |
| Traceback (most recent call last): |
| ... |
| TemplateError: {{ inside expression at line 1 column 10 |
| |
| """ |
| in_expr = False |
| chunks = [] |
| last = 0 |
| last_pos = (1, 1) |
| for match in token_re.finditer(s): |
| expr = match.group(0) |
| pos = find_position(s, match.end()) |
| if expr == '{{' and in_expr: |
| raise TemplateError('{{ inside expression', position=pos, |
| name=name) |
| elif expr == '}}' and not in_expr: |
| raise TemplateError('}} outside expression', position=pos, |
| name=name) |
| if expr == '{{': |
| part = s[last:match.start()] |
| if part: |
| chunks.append(part) |
| in_expr = True |
| else: |
| chunks.append((s[last:match.start()], last_pos)) |
| in_expr = False |
| last = match.end() |
| last_pos = pos |
| if in_expr: |
| raise TemplateError('No }} to finish last expression', |
| name=name, position=last_pos) |
| part = s[last:] |
| if part: |
| chunks.append(part) |
| if trim_whitespace: |
| chunks = trim_lex(chunks) |
| return chunks |
| |
| statement_re = re.compile(r'^(?:if |elif |else |for |py:)') |
| single_statements = ['endif', 'endfor', 'continue', 'break'] |
| trail_whitespace_re = re.compile(r'\n[\t ]*$') |
| lead_whitespace_re = re.compile(r'^[\t ]*\n') |
| |
| def trim_lex(tokens): |
| r""" |
| Takes a lexed set of tokens, and removes whitespace when there is |
| a directive on a line by itself: |
| |
| >>> tokens = lex('{{if x}}\nx\n{{endif}}\ny', trim_whitespace=False) |
| >>> tokens |
| [('if x', (1, 3)), '\nx\n', ('endif', (3, 3)), '\ny'] |
| >>> trim_lex(tokens) |
| [('if x', (1, 3)), 'x\n', ('endif', (3, 3)), 'y'] |
| """ |
| for i in range(len(tokens)): |
| current = tokens[i] |
| if isinstance(tokens[i], six.string_types): |
| # we don't trim this |
| continue |
| item = current[0] |
| if not statement_re.search(item) and item not in single_statements: |
| continue |
| if not i: |
| prev = '' |
| else: |
| prev = tokens[i-1] |
| if i+1 >= len(tokens): |
| next = '' |
| else: |
| next = tokens[i+1] |
| if (not isinstance(next, six.string_types) |
| or not isinstance(prev, six.string_types)): |
| continue |
| if ((not prev or trail_whitespace_re.search(prev)) |
| and (not next or lead_whitespace_re.search(next))): |
| if prev: |
| m = trail_whitespace_re.search(prev) |
| # +1 to leave the leading \n on: |
| prev = prev[:m.start()+1] |
| tokens[i-1] = prev |
| if next: |
| m = lead_whitespace_re.search(next) |
| next = next[m.end():] |
| tokens[i+1] = next |
| return tokens |
| |
| |
| def find_position(string, index): |
| """Given a string and index, return (line, column)""" |
| leading = string[:index].splitlines() |
| return (len(leading), len(leading[-1])+1) |
| |
| def parse(s, name=None): |
| r""" |
| Parses a string into a kind of AST |
| |
| >>> parse('{{x}}') |
| [('expr', (1, 3), 'x')] |
| >>> parse('foo') |
| ['foo'] |
| >>> parse('{{if x}}test{{endif}}') |
| [('cond', (1, 3), ('if', (1, 3), 'x', ['test']))] |
| >>> parse('series->{{for x in y}}x={{x}}{{endfor}}') |
| ['series->', ('for', (1, 11), ('x',), 'y', ['x=', ('expr', (1, 27), 'x')])] |
| >>> parse('{{for x, y in z:}}{{continue}}{{endfor}}') |
| [('for', (1, 3), ('x', 'y'), 'z', [('continue', (1, 21))])] |
| >>> parse('{{py:x=1}}') |
| [('py', (1, 3), 'x=1')] |
| >>> parse('{{if x}}a{{elif y}}b{{else}}c{{endif}}') |
| [('cond', (1, 3), ('if', (1, 3), 'x', ['a']), ('elif', (1, 12), 'y', ['b']), ('else', (1, 23), None, ['c']))] |
| |
| Some exceptions:: |
| |
| >>> parse('{{continue}}') |
| Traceback (most recent call last): |
| ... |
| TemplateError: continue outside of for loop at line 1 column 3 |
| >>> parse('{{if x}}foo') |
| Traceback (most recent call last): |
| ... |
| TemplateError: No {{endif}} at line 1 column 3 |
| >>> parse('{{else}}') |
| Traceback (most recent call last): |
| ... |
| TemplateError: else outside of an if block at line 1 column 3 |
| >>> parse('{{if x}}{{for x in y}}{{endif}}{{endfor}}') |
| Traceback (most recent call last): |
| ... |
| TemplateError: Unexpected endif at line 1 column 25 |
| >>> parse('{{if}}{{endif}}') |
| Traceback (most recent call last): |
| ... |
| TemplateError: if with no expression at line 1 column 3 |
| >>> parse('{{for x y}}{{endfor}}') |
| Traceback (most recent call last): |
| ... |
| TemplateError: Bad for (no "in") in 'x y' at line 1 column 3 |
| >>> parse('{{py:x=1\ny=2}}') |
| Traceback (most recent call last): |
| ... |
| TemplateError: Multi-line py blocks must start with a newline at line 1 column 3 |
| """ |
| tokens = lex(s, name=name) |
| result = [] |
| while tokens: |
| next, tokens = parse_expr(tokens, name) |
| result.append(next) |
| return result |
| |
| def parse_expr(tokens, name, context=()): |
| if isinstance(tokens[0], six.string_types): |
| return tokens[0], tokens[1:] |
| expr, pos = tokens[0] |
| expr = expr.strip() |
| if expr.startswith('py:'): |
| expr = expr[3:].lstrip(' \t') |
| if expr.startswith('\n'): |
| expr = expr[1:] |
| else: |
| if '\n' in expr: |
| raise TemplateError( |
| 'Multi-line py blocks must start with a newline', |
| position=pos, name=name) |
| return ('py', pos, expr), tokens[1:] |
| elif expr in ('continue', 'break'): |
| if 'for' not in context: |
| raise TemplateError( |
| 'continue outside of for loop', |
| position=pos, name=name) |
| return (expr, pos), tokens[1:] |
| elif expr.startswith('if '): |
| return parse_cond(tokens, name, context) |
| elif (expr.startswith('elif ') |
| or expr == 'else'): |
| raise TemplateError( |
| '%s outside of an if block' % expr.split()[0], |
| position=pos, name=name) |
| elif expr in ('if', 'elif', 'for'): |
| raise TemplateError( |
| '%s with no expression' % expr, |
| position=pos, name=name) |
| elif expr in ('endif', 'endfor'): |
| raise TemplateError( |
| 'Unexpected %s' % expr, |
| position=pos, name=name) |
| elif expr.startswith('for '): |
| return parse_for(tokens, name, context) |
| elif expr.startswith('default '): |
| return parse_default(tokens, name, context) |
| elif expr.startswith('#'): |
| return ('comment', pos, tokens[0][0]), tokens[1:] |
| return ('expr', pos, tokens[0][0]), tokens[1:] |
| |
| def parse_cond(tokens, name, context): |
| start = tokens[0][1] |
| pieces = [] |
| context = context + ('if',) |
| while 1: |
| if not tokens: |
| raise TemplateError( |
| 'Missing {{endif}}', |
| position=start, name=name) |
| if (isinstance(tokens[0], tuple) |
| and tokens[0][0] == 'endif'): |
| return ('cond', start) + tuple(pieces), tokens[1:] |
| next, tokens = parse_one_cond(tokens, name, context) |
| pieces.append(next) |
| |
| def parse_one_cond(tokens, name, context): |
| (first, pos), tokens = tokens[0], tokens[1:] |
| content = [] |
| if first.endswith(':'): |
| first = first[:-1] |
| if first.startswith('if '): |
| part = ('if', pos, first[3:].lstrip(), content) |
| elif first.startswith('elif '): |
| part = ('elif', pos, first[5:].lstrip(), content) |
| elif first == 'else': |
| part = ('else', pos, None, content) |
| else: |
| assert 0, "Unexpected token %r at %s" % (first, pos) |
| while 1: |
| if not tokens: |
| raise TemplateError( |
| 'No {{endif}}', |
| position=pos, name=name) |
| if (isinstance(tokens[0], tuple) |
| and (tokens[0][0] == 'endif' |
| or tokens[0][0].startswith('elif ') |
| or tokens[0][0] == 'else')): |
| return part, tokens |
| next, tokens = parse_expr(tokens, name, context) |
| content.append(next) |
| |
| def parse_for(tokens, name, context): |
| first, pos = tokens[0] |
| tokens = tokens[1:] |
| context = ('for',) + context |
| content = [] |
| assert first.startswith('for ') |
| if first.endswith(':'): |
| first = first[:-1] |
| first = first[3:].strip() |
| match = in_re.search(first) |
| if not match: |
| raise TemplateError( |
| 'Bad for (no "in") in %r' % first, |
| position=pos, name=name) |
| vars = first[:match.start()] |
| if '(' in vars: |
| raise TemplateError( |
| 'You cannot have () in the variable section of a for loop (%r)' |
| % vars, position=pos, name=name) |
| vars = tuple([ |
| v.strip() for v in first[:match.start()].split(',') |
| if v.strip()]) |
| expr = first[match.end():] |
| while 1: |
| if not tokens: |
| raise TemplateError( |
| 'No {{endfor}}', |
| position=pos, name=name) |
| if (isinstance(tokens[0], tuple) |
| and tokens[0][0] == 'endfor'): |
| return ('for', pos, vars, expr, content), tokens[1:] |
| next, tokens = parse_expr(tokens, name, context) |
| content.append(next) |
| |
| def parse_default(tokens, name, context): |
| first, pos = tokens[0] |
| assert first.startswith('default ') |
| first = first.split(None, 1)[1] |
| parts = first.split('=', 1) |
| if len(parts) == 1: |
| raise TemplateError( |
| "Expression must be {{default var=value}}; no = found in %r" % first, |
| position=pos, name=name) |
| var = parts[0].strip() |
| if ',' in var: |
| raise TemplateError( |
| "{{default x, y = ...}} is not supported", |
| position=pos, name=name) |
| if not var_re.search(var): |
| raise TemplateError( |
| "Not a valid variable name for {{default}}: %r" |
| % var, position=pos, name=name) |
| expr = parts[1].strip() |
| return ('default', pos, var, expr), tokens[1:] |
| |
| _fill_command_usage = """\ |
| %prog [OPTIONS] TEMPLATE arg=value |
| |
| Use py:arg=value to set a Python value; otherwise all values are |
| strings. |
| """ |
| |
| def fill_command(args=None): |
| import sys, optparse, pkg_resources, os |
| if args is None: |
| args = sys.argv[1:] |
| dist = pkg_resources.get_distribution('Paste') |
| parser = optparse.OptionParser( |
| version=str(dist), |
| usage=_fill_command_usage) |
| parser.add_option( |
| '-o', '--output', |
| dest='output', |
| metavar="FILENAME", |
| help="File to write output to (default stdout)") |
| parser.add_option( |
| '--html', |
| dest='use_html', |
| action='store_true', |
| help="Use HTML style filling (including automatic HTML quoting)") |
| parser.add_option( |
| '--env', |
| dest='use_env', |
| action='store_true', |
| help="Put the environment in as top-level variables") |
| options, args = parser.parse_args(args) |
| if len(args) < 1: |
| print('You must give a template filename') |
| print(dir(parser)) |
| assert 0 |
| template_name = args[0] |
| args = args[1:] |
| vars = {} |
| if options.use_env: |
| vars.update(os.environ) |
| for value in args: |
| if '=' not in value: |
| print('Bad argument: %r' % value) |
| sys.exit(2) |
| name, value = value.split('=', 1) |
| if name.startswith('py:'): |
| name = name[:3] |
| value = eval(value) |
| vars[name] = value |
| if template_name == '-': |
| template_content = sys.stdin.read() |
| template_name = '<stdin>' |
| else: |
| f = open(template_name, 'rb') |
| template_content = f.read() |
| f.close() |
| if options.use_html: |
| TemplateClass = HTMLTemplate |
| else: |
| TemplateClass = Template |
| template = TemplateClass(template_content, name=template_name) |
| result = template.substitute(vars) |
| if options.output: |
| f = open(options.output, 'wb') |
| f.write(result) |
| f.close() |
| else: |
| sys.stdout.write(result) |
| |
| if __name__ == '__main__': |
| from paste.util.template import fill_command |
| fill_command() |
| |
| |
