starting to put in the module loading crap...
diff --git a/doc/build/read_markdown.py b/doc/build/read_markdown.py index eab5b6f..cbd1a8c 100644 --- a/doc/build/read_markdown.py +++ b/doc/build/read_markdown.py
@@ -148,11 +148,6 @@ tag.text = code.text code.append(tag) code.text = "" - #code.tail =None - #tag.text = code.text - #code_parent = parents[code] - #code_parent[reverse_parent(code_parent, code)] = tag - def reverse_parent(parent, item): for n, i in enumerate(parent):
diff --git a/lib/mako/lookup.py b/lib/mako/lookup.py index 53d580a..44cfd2a 100644 --- a/lib/mako/lookup.py +++ b/lib/mako/lookup.py
@@ -4,11 +4,15 @@ # This module is part of Mako and is released under # the MIT License: http://www.opensource.org/licenses/mit-license.php -import posixpath -import os -from mako import exceptions +import os, stat, posixpath, re +from mako import exceptions, util from mako.template import Template +try: + import threading +except: + import dummy_threading as threading + class TemplateCollection(object): def has_template(self, uri): try: @@ -20,25 +24,75 @@ raise NotImplementedError() class TemplateLookup(TemplateCollection): - def __init__(self, directories=None, module_directory=None, filesystem_checks=True, collection_size=-1, format_exceptions=False, error_handler=None, output_encoding=None): + def __init__(self, directories=None, module_directory=None, filesystem_checks=False, collection_size=-1, format_exceptions=False, error_handler=None, output_encoding=None): self.directories = directories or [] self.module_directory = module_directory self.filesystem_checks = filesystem_checks self.collection_size = collection_size self.template_args = {'format_exceptions':format_exceptions, 'error_handler':error_handler, 'output_encoding':output_encoding} - self._collection = {} + if collection_size == -1: + self.__collection = {} + else: + self.__collection = util.LRUCache(collection_size) + self._mutex = threading.Lock() + def get_template(self, uri): try: - return self._collection[uri] - except KeyError: - for dir in self.directories: - srcfile = posixpath.join(dir, uri) - if os.access(srcfile, os.F_OK): - self._collection[uri] = Template(file(srcfile).read(), description=uri, filename=srcfile, lookup=self, **self.template_args) - return self._collection[uri] + if self.filesystem_checks: + return self.__check(uri, self.__collection[uri]) else: - raise exceptions.TemplateLookupException("Cant locate template for uri '%s'" % uri) + return self.__collection[uri] + except KeyError: + self._mutex.acquire() + try: + try: + return self.__collection[uri] + except KeyError: + for dir in self.directories: + srcfile = posixpath.join(dir, uri) + if os.access(srcfile, os.F_OK): + return self.__load(srcfile, uri) + else: + raise exceptions.TemplateLookupException("Cant locate template for uri '%s'" % uri) + finally: + self._mutex.release() + + def __ident_from_uri(self, uri): + return re.sub(r"\W", "_", uri) + + def __load(self, filename, uri): + try: + self.__collection[uri] = Template(file(filename).read(), identifier=self.__ident_from_uri(uri), description=uri, filename=filename, lookup=self, **self.template_args) + return self.__collection[uri] + except: + self.__collection.pop(uri, None) + raise + + def __check(self, uri, template): + if template.filename is None: + return template + if not os.access(template.filename, os.F_OK): + self.__collection.pop(uri, None) + raise exceptions.TemplateLookupException("Cant locate template for uri '%s'" % uri) + elif template.module._modified_time < os.stat(template.filename)[stat.ST_MTIME]: + return __load(template.filename, uri) + else: + return template + def put_string(self, uri, text): - self._collection[uri] = Template(text, lookup=self, description=uri, **self.template_args) + lock = sync.NameLock(uri) + lock.acquire() + try: + self.__collection[uri] = Template(text, lookup=self, description=uri, **self.template_args) + finally: + lock.release() def put_template(self, uri, template): - self._collection[uri] = template \ No newline at end of file + lock = sync.NameLock(uri) + lock.acquire() + try: + self.__collection[uri] = template + finally: + lock.release() + + + \ No newline at end of file
diff --git a/lib/mako/template.py b/lib/mako/template.py index cb071d0..34e9bcf 100644 --- a/lib/mako/template.py +++ b/lib/mako/template.py
@@ -11,7 +11,7 @@ from mako.codegen import Compiler from mako import runtime from mako import util -import imp, time, weakref +import imp, time, weakref, tempfile, shutil _modules = weakref.WeakValueDictionary() _inmemory_templates = weakref.WeakValueDictionary() @@ -23,14 +23,11 @@ class Template(object): """a compiled template""" - def __init__(self, text=None, module=None, identifier=None, description=None, filename=None, format_exceptions=False, error_handler=None, lookup=None, output_encoding=None): + def __init__(self, text=None,identifier=None, description=None, filename=None, format_exceptions=False, error_handler=None, lookup=None, output_encoding=None, module_directory=None): """construct a new Template instance using either literal template text, or a previously loaded template module text - textual template source, or None if a module is to be provided - module - a Python module, such as loaded via __import__ or similar. the module should contain at least one - function render(context) that renders with the given context. - identifier - the "id" of this template. defaults to the identifier of the given module, or for text the hex string of this Template's object id @@ -45,9 +42,21 @@ _inmemory_templates[module.__name__] = self self._code = code self._source = text + elif filename is not None: + if module_directory is not None: + path = posixpath.join(module_directory, identifier + ".py") + if not os.access(path, os.F_OK): + util.verify_directory(module_directory) + _compile_module(text, identifier, filename, module_directory) + module = imp.load_source(self.identifier, path, file(path)) + del sys.modules[self.identifier] + else: + (code, module) = _compile_text(file(filename), self.identifier, filename) + self._source = None + self._code = code else: - self._source = None - self._code = None + raise exceptions.RuntimeException("Template requires text or filename") + self.module = module self.description = description self.filename = filename @@ -96,40 +105,54 @@ def _compile_text(text, identifier, filename): node = Lexer(text, filename).parse() source = Compiler(node, filename).render() - if filename is not None: - file(filename + ".py", "w").write(source) - else: - print source +# if filename is not None: +# file(filename + ".py", "w").write(source) +# else: +# print source cid = identifier module = imp.new_module(cid) code = compile(source, cid + " " + str(filename), 'exec') exec code in module.__dict__, module.__dict__ return (source, module) +def _compile_module(text, identifier, filename, outputpath): + + dest = tempfile.NamedTemporaryFile() + + node = Lexer(text, filename).parse() + source = Compiler(node, filename).render() + dest.write(source) + dest.close() + shutil.move(dest.name, outputpath) + def _get_template_source(callable_): """return the source code for the template that produced the given rendering callable""" name = callable_.func_globals['__name__'] try: template = _inmemory_templates[name] - return template._source + if template._source is not None: + return template._source except KeyError: - module = _modules[name].module - filename = module._template_filename - if filename is None: - if not filename: - raise exceptions.RuntimeException("Cant get source code or template filename for template: %s" % name) - return file(filename).read() + pass + module = _modules[name].module + filename = module._template_filename + if filename is None: + if not filename: + raise exceptions.RuntimeException("Cant get source code or template filename for template: %s" % name) + return file(filename).read() def _get_module_source(callable_): name = callable_.func_globals['__name__'] try: template = _inmemory_templates[name] - return template._code + if template._code is not None: + return template._code except KeyError: - module = _modules[name].module - filename = module.__file__ - if filename is None: - if not filename: - raise exceptions.RuntimeException("Cant get module source code or module filename for template: %s" % name) - return file(filename).read() + pass + module = _modules[name].module + filename = module.__file__ + if filename is None: + if not filename: + raise exceptions.RuntimeException("Cant get module source code or module filename for template: %s" % name) + return file(filename).read()
diff --git a/lib/mako/util.py b/lib/mako/util.py index 0560d8a..a017830 100644 --- a/lib/mako/util.py +++ b/lib/mako/util.py
@@ -15,7 +15,26 @@ except: from StringIO import StringIO +import weakref, os +try: + import threading + import thread +except ImportError: + import dummy_threading as threading + import dummy_thread as thread + +def verify_directory(dir): + """create and/or verify a filesystem directory.""" + tries = 0 + while not os.access(dir, os.F_OK): + try: + tries += 1 + os.makedirs(dir, 0750) + except: + if tries > 5: + raise + class FastEncodingBuffer(object): """a very rudimentary buffer that is faster than StringIO, but doesnt crash on unicode data like cStringIO.""" def __init__(self, encoding=None): @@ -29,3 +48,111 @@ else: return u''.join(self.data) +class ThreadLocalRegistry(object): + """a registry that stores instances keyed to the current thread. if a requested + instance does not exist for a particular thread, it is created by a creation function. + """ + def __init__(self, createfunc): + self.__createfunc = createfunc + self.__scopefunc = thread.get_ident + self.__registry = {} + def __call__(self): + key = self.__scopefunc() + try: + return self.__registry[key] + except KeyError: + return self.__registry.setdefault(key, self.__createfunc()) + def set(self, obj): + self.registry[self.__scopefunc()] = obj + def clear(self): + try: + del self.registry[self.__scopefunc()] + except KeyError: + pass + +class LRUCache(dict): + """A dictionary-like object that stores only a certain number of items, and + discards its least recently used item when full. + + this is Chris Lenz' cleanup of Mike Bayer's version from Myghty. + """ + + class _Item(object): + def __init__(self, key, value): + self.previous = self.next = None + self.key = key + self.value = value + def __repr__(self): + return repr(self.value) + + def __init__(self, capacity): + self._dict = dict() + self.capacity = capacity + self.head = None + self.tail = None + + def __contains__(self, key): + return key in self._dict + + def __iter__(self): + cur = self.head + while cur: + yield cur.key + cur = cur.next + + def __len__(self): + return len(self._dict) + + def __getitem__(self, key): + item = self._dict[key] + self._update_item(item) + return item.value + + def __setitem__(self, key, value): + item = self._dict.get(key) + if item is None: + item = self._Item(key, value) + self._dict[key] = item + self._insert_item(item) + else: + item.value = value + self._update_item(item) + self._manage_size() + + def __repr__(self): + return repr(self._dict) + + def _insert_item(self, item): + item.previous = None + item.next = self.head + if self.head is not None: + self.head.previous = item + else: + self.tail = item + self.head = item + self._manage_size() + + def _manage_size(self): + while len(self._dict) > self.capacity: + olditem = self._dict[self.tail.key] + del self._dict[self.tail.key] + if self.tail != self.head: + self.tail = self.tail.previous + self.tail.next = None + else: + self.head = self.tail = None + + def _update_item(self, item): + if self.head == item: + return + + previous = item.previous + previous.next = item.next + if item.next is not None: + item.next.previous = previous + else: + self.tail = previous + + item.previous = None + item.next = self.head + self.head.previous = self.head = item