Fixed #6262 -- Added a cached template loader, and modified existing template loaders and tag to be cacheable. Thanks to Mike Malone for the patch.

git-svn-id: http://code.djangoproject.com/svn/django/trunk@11862 bcc190cf-cafb-0310-a4f2-bffc1f526a37

AUTHORS

@@ -288,6 +288,7 @@ answer newbie questions, and generally made Django that much better:
     Martin Mahner <http://www.mahner.org/>
     Matt McClanahan <http://mmcc.cx/>
     Frantisek Malina <vizualbod@vizualbod.com>
+    Mike Malone <mjmalone@gmail.com>
     Martin Maney <http://www.chipy.org/Martin_Maney>
     masonsimon+django@gmail.com
     Manuzhai

django/conf/global_settings.py

@@ -158,9 +158,9 @@ TEMPLATE_DIRS = ()
 # See the comments in django/core/template/loader.py for interface
 # documentation.
 TEMPLATE_LOADERS = (
-    'django.template.loaders.filesystem.load_template_source',
-    'django.template.loaders.app_directories.load_template_source',
-#     'django.template.loaders.eggs.load_template_source',
+    'django.template.loaders.filesystem.Loader',
+    'django.template.loaders.app_directories.Loader',
+#     'django.template.loaders.eggs.Loader',
 )
 
 # List of processors used by RequestContext to populate the context.

django/conf/project_template/settings.py

@@ -52,9 +52,9 @@ SECRET_KEY = ''
 
 # List of callables that know how to import templates from various sources.
 TEMPLATE_LOADERS = (
-    'django.template.loaders.filesystem.load_template_source',
-    'django.template.loaders.app_directories.load_template_source',
-#     'django.template.loaders.eggs.load_template_source',
+    'django.template.loaders.filesystem.Loader',
+    'django.template.loaders.app_directories.Loader',
+#     'django.template.loaders.eggs.Loader',
 )
 
 MIDDLEWARE_CLASSES = (

django/template/__init__.py

@@ -173,9 +173,16 @@ class Template(object):
             for subnode in node:
                 yield subnode
 
+    def _render(self, context):
+        return self.nodelist.render(context)
+
     def render(self, context):
         "Display stage -- can be called many times"
-        return self.nodelist.render(context)
+        context.render_context.push()
+        try:
+            return self._render(context)
+        finally:
+            context.render_context.pop()
 
 def compile_string(template_string, origin):
     "Compiles template_string into NodeList ready for rendering"

django/template/context.py

@@ -12,45 +12,42 @@ class ContextPopException(Exception):
     "pop() has been called more times than push()"
     pass
 
-class Context(object):
-    "A stack container for variable context"
-    def __init__(self, dict_=None, autoescape=True, current_app=None):
+class BaseContext(object):
+    def __init__(self, dict_=None):
         dict_ = dict_ or {}
         self.dicts = [dict_]
-        self.autoescape = autoescape
-        self.current_app = current_app
 
     def __repr__(self):
         return repr(self.dicts)
 
     def __iter__(self):
-        for d in self.dicts:
+        for d in reversed(self.dicts):
             yield d
 
     def push(self):
         d = {}
-        self.dicts = [d] + self.dicts
+        self.dicts.append(d)
         return d
 
     def pop(self):
         if len(self.dicts) == 1:
             raise ContextPopException
-        return self.dicts.pop(0)
+        return self.dicts.pop()
 
     def __setitem__(self, key, value):
         "Set a variable in the current context"
-        self.dicts[0][key] = value
+        self.dicts[-1][key] = value
 
     def __getitem__(self, key):
         "Get a variable's value, starting at the current context and going upward"
-        for d in self.dicts:
+        for d in reversed(self.dicts):
             if key in d:
                 return d[key]
         raise KeyError(key)
 
     def __delitem__(self, key):
         "Delete a variable from the current context"
-        del self.dicts[0][key]
+        del self.dicts[-1][key]
 
     def has_key(self, key):
         for d in self.dicts:
@@ -58,21 +55,58 @@ class Context(object):
                 return True
         return False
 
-    __contains__ = has_key
+    def __contains__(self, key):
+        return self.has_key(key)
 
     def get(self, key, otherwise=None):
-        for d in self.dicts:
+        for d in reversed(self.dicts):
             if key in d:
                 return d[key]
         return otherwise
 
+class Context(BaseContext):
+    "A stack container for variable context"
+    def __init__(self, dict_=None, autoescape=True, current_app=None):
+        self.autoescape = autoescape
+        self.current_app = current_app
+        self.render_context = RenderContext()
+        super(Context, self).__init__(dict_)
+
     def update(self, other_dict):
         "Like dict.update(). Pushes an entire dictionary's keys and values onto the context."
         if not hasattr(other_dict, '__getitem__'):
             raise TypeError('other_dict must be a mapping (dictionary-like) object.')
-        self.dicts = [other_dict] + self.dicts
+        self.dicts.append(other_dict)
         return other_dict
 
+class RenderContext(BaseContext):
+    """
+    A stack container for storing Template state.
+
+    RenderContext simplifies the implementation of template Nodes by providing a
+    safe place to store state between invocations of a node's `render` method.
+
+    The RenderContext also provides scoping rules that are more sensible for
+    'template local' variables. The render context stack is pushed before each
+    template is rendered, creating a fresh scope with nothing in it. Name
+    resolution fails if a variable is not found at the top of the RequestContext
+    stack. Thus, variables are local to a specific template and don't affect the
+    rendering of other templates as they would if they were stored in the normal
+    template context.
+    """
+    def __iter__(self):
+        for d in self.dicts[-1]:
+            yield d
+
+    def has_key(self, key):
+        return key in self.dicts[-1]
+
+    def get(self, key, otherwise=None):
+        d = self.dicts[-1]
+        if key in d:
+            return d[key]
+        return otherwise
+
 # This is a function rather than module-level procedural code because we only
 # want it to execute if somebody uses RequestContext.
 def get_standard_processors():

django/template/defaulttags.py

@@ -57,11 +57,14 @@ class CsrfTokenNode(Node):
 
 class CycleNode(Node):
     def __init__(self, cyclevars, variable_name=None):
-        self.cycle_iter = itertools_cycle(cyclevars)
+        self.cyclevars = cyclevars
         self.variable_name = variable_name
 
     def render(self, context):
-        value = self.cycle_iter.next().resolve(context)
+        if self not in context.render_context:
+            context.render_context[self] = itertools_cycle(self.cyclevars)
+        cycle_iter = context.render_context[self]
+        value = cycle_iter.next().resolve(context)
         if self.variable_name:
             context[self.variable_name] = value
         return value

django/template/loader.py

@@ -27,6 +27,36 @@ from django.conf import settings
 
 template_source_loaders = None
 
+class BaseLoader(object):
+    is_usable = False
+
+    def __init__(self, *args, **kwargs):
+        pass
+
+    def __call__(self, template_name, template_dirs=None):
+        return self.load_template(template_name, template_dirs)
+
+    def load_template(self, template_name, template_dirs=None):
+        source, origin = self.load_template_source(template_name, template_dirs)
+        template = get_template_from_string(source, name=template_name)
+        return template, origin
+
+    def load_template_source(self, template_name, template_dirs=None):
+        """
+        Returns a tuple containing the source and origin for the given template
+        name.
+
+        """
+        raise NotImplementedError
+
+    def reset(self):
+        """
+        Resets any state maintained by the loader instance (e.g., cached
+        templates or cached loader modules).
+
+        """
+        pass
+
 class LoaderOrigin(Origin):
     def __init__(self, display_name, loader, name, dirs):
         super(LoaderOrigin, self).__init__(display_name)
@@ -41,29 +71,50 @@ def make_origin(display_name, loader, name, dirs):
     else:
         return None
 
-def find_template_source(name, dirs=None):
+def find_template_loader(loader):
+    if hasattr(loader, '__iter__'):
+        loader, args = loader[0], loader[1:]
+    else:
+        args = []
+    if isinstance(loader, basestring):
+        module, attr = loader.rsplit('.', 1)
+        try:
+            mod = import_module(module)
+        except ImportError:
+            raise ImproperlyConfigured('Error importing template source loader %s: "%s"' % (loader, e))
+        try:
+            TemplateLoader = getattr(mod, attr)
+        except AttributeError, e:
+            raise ImproperlyConfigured('Error importing template source loader %s: "%s"' % (loader, e))
+
+        if hasattr(TemplateLoader, 'load_template_source'):
+            func = TemplateLoader(*args)
+        else:
+            # Try loading module the old way - string is full path to callable
+            if args:
+                raise ImproperlyConfigured("Error importing template source loader %s - can't pass arguments to function-based loader." % loader)
+            func = TemplateLoader
+
+        if not func.is_usable:
+            import warnings
+            warnings.warn("Your TEMPLATE_LOADERS setting includes %r, but your Python installation doesn't support that type of template loading. Consider removing that line from TEMPLATE_LOADERS." % loader)
+            return None
+        else:
+            return func
+    else:
+        raise ImproperlyConfigured('Loader does not define a "load_template" callable template source loader')
+
+def find_template(name, dirs=None):
     # Calculate template_source_loaders the first time the function is executed
     # because putting this logic in the module-level namespace may cause
     # circular import errors. See Django ticket #1292.
     global template_source_loaders
     if template_source_loaders is None:
         loaders = []
-        for path in settings.TEMPLATE_LOADERS:
-            i = path.rfind('.')
-            module, attr = path[:i], path[i+1:]
-            try:
-                mod = import_module(module)
-            except ImportError, e:
-                raise ImproperlyConfigured, 'Error importing template source loader %s: "%s"' % (module, e)
-            try:
-                func = getattr(mod, attr)
-            except AttributeError:
-                raise ImproperlyConfigured, 'Module "%s" does not define a "%s" callable template source loader' % (module, attr)
-            if not func.is_usable:
-                import warnings
-                warnings.warn("Your TEMPLATE_LOADERS setting includes %r, but your Python installation doesn't support that type of template loading. Consider removing that line from TEMPLATE_LOADERS." % path)
-            else:
-                loaders.append(func)
+        for loader_name in settings.TEMPLATE_LOADERS:
+            loader = find_template_loader(loader_name)
+            if loader is not None:
+                loaders.append(loader)
         template_source_loaders = tuple(loaders)
     for loader in template_source_loaders:
         try:
@@ -73,13 +124,27 @@ def find_template_source(name, dirs=None):
             pass
     raise TemplateDoesNotExist, name
 
+def find_template_source(name, dirs=None):
+    # For backward compatibility
+    import warnings
+    warnings.warn(
+        "`django.template.loaders.find_template_source` is deprecated; use `django.template.loaders.find_template` instead.",
+        PendingDeprecationWarning
+    )
+    template, origin = find_template(name, dirs)
+    if hasattr(template, 'render'):
+        raise Exception("Found a compiled template that is incompatible with the deprecated `django.template.loaders.find_template_source` function.")
+    return template, origin
+
 def get_template(template_name):
     """
     Returns a compiled Template object for the given template name,
     handling template inheritance recursively.
     """
-    source, origin = find_template_source(template_name)
-    template = get_template_from_string(source, origin, template_name)
+    template, origin = find_template(template_name)
+    if not hasattr(template, 'render'):
+        # template needs to be compiled
+        template = get_template_from_string(template, origin, template_name)
     return template
 
 def get_template_from_string(source, origin=None, name=None):

django/template/loader_tags.py

@@ -1,14 +1,43 @@
 from django.template import TemplateSyntaxError, TemplateDoesNotExist, Variable
 from django.template import Library, Node, TextNode
-from django.template.loader import get_template, get_template_from_string, find_template_source
+from django.template.loader import get_template
 from django.conf import settings
 from django.utils.safestring import mark_safe
 
 register = Library()
 
+BLOCK_CONTEXT_KEY = 'block_context'
+
 class ExtendsError(Exception):
     pass
 
+class BlockContext(object):
+    def __init__(self):
+        # Dictionary of FIFO queues.
+        self.blocks = {}
+
+    def add_blocks(self, blocks):
+        for name, block in blocks.iteritems():
+            if name in self.blocks:
+                self.blocks[name].insert(0, block)
+            else:
+                self.blocks[name] = [block]
+
+    def pop(self, name):
+        try:
+            return self.blocks[name].pop()
+        except (IndexError, KeyError):
+            return None
+
+    def push(self, name, block):
+        self.blocks[name].append(block)
+
+    def get_block(self, name):
+        try:
+            return self.blocks[name][-1]
+        except (IndexError, KeyError):
+            return None
+
 class BlockNode(Node):
     def __init__(self, name, nodelist, parent=None):
         self.name, self.nodelist, self.parent = name, nodelist, parent
@@ -17,25 +46,32 @@ class BlockNode(Node):
         return "<Block Node: %s. Contents: %r>" % (self.name, self.nodelist)
 
     def render(self, context):
+        block_context = context.render_context.get(BLOCK_CONTEXT_KEY)
         context.push()
-        # Save context in case of block.super().
-        self.context = context
-        context['block'] = self
-        result = self.nodelist.render(context)
+        if block_context is None:
+            context['block'] = self
+            result = self.nodelist.render(context)
+        else:
+            push = block = block_context.pop(self.name)
+            if block is None:
+                block = self
+            # Create new block so we can store context without thread-safety issues.
+            block = BlockNode(block.name, block.nodelist)
+            block.context = context
+            context['block'] = block
+            result = block.nodelist.render(context)
+            if push is not None:
+                block_context.push(self.name, push)
         context.pop()
         return result
 
     def super(self):
-        if self.parent:
-            return mark_safe(self.parent.render(self.context))
+        render_context = self.context.render_context
+        if (BLOCK_CONTEXT_KEY in render_context and
+            render_context[BLOCK_CONTEXT_KEY].get_block(self.name) is not None):
+            return mark_safe(self.render(self.context))
         return ''
 
-    def add_parent(self, nodelist):
-        if self.parent:
-            self.parent.add_parent(nodelist)
-        else:
-            self.parent = BlockNode(self.name, nodelist)
-
 class ExtendsNode(Node):
     must_be_first = True
 
@@ -43,6 +79,7 @@ class ExtendsNode(Node):
         self.nodelist = nodelist
         self.parent_name, self.parent_name_expr = parent_name, parent_name_expr
         self.template_dirs = template_dirs
+        self.blocks = dict([(n.name, n) for n in nodelist.get_nodes_by_type(BlockNode)])
 
     def __repr__(self):
         if self.parent_name_expr:
@@ -61,40 +98,34 @@ class ExtendsNode(Node):
         if hasattr(parent, 'render'):
             return parent # parent is a Template object
         try:
-            source, origin = find_template_source(parent, self.template_dirs)
+            return get_template(parent)
         except TemplateDoesNotExist:
             raise TemplateSyntaxError, "Template %r cannot be extended, because it doesn't exist" % parent
-        else:
-            return get_template_from_string(source, origin, parent)
 
     def render(self, context):
         compiled_parent = self.get_parent(context)
-        parent_blocks = dict([(n.name, n) for n in compiled_parent.nodelist.get_nodes_by_type(BlockNode)])
-        for block_node in self.nodelist.get_nodes_by_type(BlockNode):
-            # Check for a BlockNode with this node's name, and replace it if found.
-            try:
-                parent_block = parent_blocks[block_node.name]
-            except KeyError:
-                # This BlockNode wasn't found in the parent template, but the
-                # parent block might be defined in the parent's *parent*, so we
-                # add this BlockNode to the parent's ExtendsNode nodelist, so
-                # it'll be checked when the parent node's render() is called.
-
-                # Find out if the parent template has a parent itself
-                for node in compiled_parent.nodelist:
-                    if not isinstance(node, TextNode):
-                        # If the first non-text node is an extends, handle it.
-                        if isinstance(node, ExtendsNode):
-                            node.nodelist.append(block_node)
-                        # Extends must be the first non-text node, so once you find
-                        # the first non-text node you can stop looking. 
-                        break
-            else:
-                # Keep any existing parents and add a new one. Used by BlockNode.
-                parent_block.parent = block_node.parent
-                parent_block.add_parent(parent_block.nodelist)
-                parent_block.nodelist = block_node.nodelist
-        return compiled_parent.render(context)
+
+        if BLOCK_CONTEXT_KEY not in context.render_context:
+            context.render_context[BLOCK_CONTEXT_KEY] = BlockContext()
+        block_context = context.render_context[BLOCK_CONTEXT_KEY]
+
+        # Add the block nodes from this node to the block context
+        block_context.add_blocks(self.blocks)
+
+        # If this block's parent doesn't have an extends node it is the root,
+        # and its block nodes also need to be added to the block context.
+        for node in compiled_parent.nodelist:
+            # The ExtendsNode has to be the first non-text node.
+            if not isinstance(node, TextNode):
+                if not isinstance(node, ExtendsNode):
+                    blocks = dict([(n.name, n) for n in
+                                   compiled_parent.nodelist.get_nodes_by_type(BlockNode)])
+                    block_context.add_blocks(blocks)
+                break
+
+        # Call Template._render explicitly so the parser context stays
+        # the same.
+        return compiled_parent._render(context)
 
 class ConstantIncludeNode(Node):
     def __init__(self, template_path):

django/template/loaders/app_directories.py

@@ -9,6 +9,7 @@ import sys
 from django.conf import settings
 from django.core.exceptions import ImproperlyConfigured
 from django.template import TemplateDoesNotExist
+from django.template.loader import BaseLoader
 from django.utils._os import safe_join
 from django.utils.importlib import import_module
 
@@ -27,29 +28,47 @@ for app in settings.INSTALLED_APPS:
 # It won't change, so convert it to a tuple to save memory.
 app_template_dirs = tuple(app_template_dirs)
 
-def get_template_sources(template_name, template_dirs=None):
-    """
-    Returns the absolute paths to "template_name", when appended to each
-    directory in "template_dirs". Any paths that don't lie inside one of the
-    template dirs are excluded from the result set, for security reasons.
-    """
-    if not template_dirs:
-        template_dirs = app_template_dirs
-    for template_dir in template_dirs:
-        try:
-            yield safe_join(template_dir, template_name)
-        except UnicodeDecodeError:
-            # The template dir name was a bytestring that wasn't valid UTF-8.
-            raise
-        except ValueError:
-            # The joined path was located outside of template_dir.
-            pass
+class Loader(BaseLoader):
+    is_usable = True
+
+    def get_template_sources(self, template_name, template_dirs=None):
+        """
+        Returns the absolute paths to "template_name", when appended to each
+        directory in "template_dirs". Any paths that don't lie inside one of the
+        template dirs are excluded from the result set, for security reasons.
+        """
+        if not template_dirs:
+            template_dirs = app_template_dirs
+        for template_dir in template_dirs:
+            try:
+                yield safe_join(template_dir, template_name)
+            except UnicodeDecodeError:
+                # The template dir name was a bytestring that wasn't valid UTF-8.
+                raise
+            except ValueError:
+                # The joined path was located outside of template_dir.
+                pass
+
+    def load_template_source(self, template_name, template_dirs=None):
+        for filepath in self.get_template_sources(template_name, template_dirs):
+            try:
+                file = open(filepath)
+                try:
+                    return (file.read().decode(settings.FILE_CHARSET), filepath)
+                finally:
+                    file.close()
+            except IOError:
+                pass
+        raise TemplateDoesNotExist, template_name
+
+_loader = Loader()
 
 def load_template_source(template_name, template_dirs=None):
-    for filepath in get_template_sources(template_name, template_dirs):
-        try:
-            return (open(filepath).read().decode(settings.FILE_CHARSET), filepath)
-        except IOError:
-            pass
-    raise TemplateDoesNotExist, template_name
+    # For backwards compatibility
+    import warnings
+    warnings.warn(
+        "'django.template.loaders.app_directories.load_template_source' is deprecated; use 'django.template.loaders.app_directories.Loader' instead.",
+        PendingDeprecationWarning
+    )
+    return _loader.load_template_source(template_name, template_dirs)
 load_template_source.is_usable = True

django/template/loaders/cached.py

@@ -0,0 +1,46 @@
+"""
+Wrapper class that takes a list of template loaders as an argument and attempts
+to load templates from them in order, caching the result.
+"""
+
+from django.template import TemplateDoesNotExist
+from django.template.loader import BaseLoader, get_template_from_string, find_template_loader, make_origin
+from django.utils.importlib import import_module
+from django.core.exceptions import ImproperlyConfigured
+
+class Loader(BaseLoader):
+    is_usable = True
+
+    def __init__(self, loaders):
+        self.template_cache = {}
+        self._loaders = loaders
+        self._cached_loaders = []
+
+    @property
+    def loaders(self):
+        # Resolve loaders on demand to avoid circular imports
+        if not self._cached_loaders:
+            for loader in self._loaders:
+                self._cached_loaders.append(find_template_loader(loader))
+        return self._cached_loaders
+
+    def find_template(self, name, dirs=None):
+        for loader in self.loaders:
+            try:
+                template, display_name = loader(name, dirs)
+                return (template, make_origin(display_name, loader, name, dirs))
+            except TemplateDoesNotExist:
+                pass
+        raise TemplateDoesNotExist, name
+
+    def load_template(self, template_name, template_dirs=None):
+        if template_name not in self.template_cache:
+            template, origin = self.find_template(template_name, template_dirs)
+            if not hasattr(template, 'render'):
+                template = get_template_from_string(template, origin, template_name)
+            self.template_cache[template_name] = (template, origin)
+        return self.template_cache[template_name]
+
+    def reset(self):
+        "Empty the template cache."
+        self.template_cache.clear()

django/template/loaders/eggs.py

@@ -6,20 +6,34 @@ except ImportError:
     resource_string = None
 
 from django.template import TemplateDoesNotExist
+from django.template.loader import BaseLoader
 from django.conf import settings
 
-def load_template_source(template_name, template_dirs=None):
-    """
-    Loads templates from Python eggs via pkg_resource.resource_string.
+class Loader(BaseLoader):
+    is_usable = resource_string is not None
+
+    def load_template_source(self, template_name, template_dirs=None):
+        """
+        Loads templates from Python eggs via pkg_resource.resource_string.
+
+        For every installed app, it tries to get the resource (app, template_name).
+        """
+        if resource_string is not None:
+            pkg_name = 'templates/' + template_name
+            for app in settings.INSTALLED_APPS:
+                try:
+                    return (resource_string(app, pkg_name).decode(settings.FILE_CHARSET), 'egg:%s:%s' % (app, pkg_name))
+                except:
+                    pass
+        raise TemplateDoesNotExist, template_name
 
-    For every installed app, it tries to get the resource (app, template_name).
-    """
-    if resource_string is not None:
-        pkg_name = 'templates/' + template_name
-        for app in settings.INSTALLED_APPS:
-            try:
-                return (resource_string(app, pkg_name).decode(settings.FILE_CHARSET), 'egg:%s:%s' % (app, pkg_name))
-            except:
-                pass
-    raise TemplateDoesNotExist, template_name
+_loader = Loader()
+
+def load_template_source(template_name, template_dirs=None):
+    import warnings
+    warnings.warn(
+        "'django.template.loaders.eggs.load_template_source' is deprecated; use 'django.template.loaders.eggs.Loader' instead.",
+        PendingDeprecationWarning
+    )
+    return _loader.load_template_source(template_name, template_dirs)
 load_template_source.is_usable = resource_string is not None

django/template/loaders/filesystem.py

@@ -4,38 +4,58 @@ Wrapper for loading templates from the filesystem.
 
 from django.conf import settings
 from django.template import TemplateDoesNotExist
+from django.template.loader import BaseLoader
 from django.utils._os import safe_join
 
-def get_template_sources(template_name, template_dirs=None):
-    """
-    Returns the absolute paths to "template_name", when appended to each
-    directory in "template_dirs". Any paths that don't lie inside one of the
-    template dirs are excluded from the result set, for security reasons.
-    """
-    if not template_dirs:
-        template_dirs = settings.TEMPLATE_DIRS
-    for template_dir in template_dirs:
-        try:
-            yield safe_join(template_dir, template_name)
-        except UnicodeDecodeError:
-            # The template dir name was a bytestring that wasn't valid UTF-8.
-            raise
-        except ValueError:
-            # The joined path was located outside of this particular
-            # template_dir (it might be inside another one, so this isn't
-            # fatal).
-            pass
+class Loader(BaseLoader):
+    is_usable = True
+
+    def get_template_sources(self, template_name, template_dirs=None):
+        """
+        Returns the absolute paths to "template_name", when appended to each
+        directory in "template_dirs". Any paths that don't lie inside one of the
+        template dirs are excluded from the result set, for security reasons.
+        """
+        if not template_dirs:
+            template_dirs = settings.TEMPLATE_DIRS
+        for template_dir in template_dirs:
+            try:
+                yield safe_join(template_dir, template_name)
+            except UnicodeDecodeError:
+                # The template dir name was a bytestring that wasn't valid UTF-8.
+                raise
+            except ValueError:
+                # The joined path was located outside of this particular
+                # template_dir (it might be inside another one, so this isn't
+                # fatal).
+                pass
+
+    def load_template_source(self, template_name, template_dirs=None):
+        tried = []
+        for filepath in self.get_template_sources(template_name, template_dirs):
+            try:
+                file = open(filepath)
+                try:
+                    return (file.read().decode(settings.FILE_CHARSET), filepath)
+                finally:
+                    file.close()
+            except IOError:
+                tried.append(filepath)
+        if tried:
+            error_msg = "Tried %s" % tried
+        else:
+            error_msg = "Your TEMPLATE_DIRS setting is empty. Change it to point to at least one template directory."
+        raise TemplateDoesNotExist, error_msg
+    load_template_source.is_usable = True
+
+_loader = Loader()
 
 def load_template_source(template_name, template_dirs=None):
-    tried = []
-    for filepath in get_template_sources(template_name, template_dirs):
-        try:
-            return (open(filepath).read().decode(settings.FILE_CHARSET), filepath)
-        except IOError:
-            tried.append(filepath)
-    if tried:
-        error_msg = "Tried %s" % tried
-    else:
-        error_msg = "Your TEMPLATE_DIRS setting is empty. Change it to point to at least one template directory."
-    raise TemplateDoesNotExist, error_msg
+    # For backwards compatibility
+    import warnings
+    warnings.warn(
+        "'django.template.loaders.filesystem.load_template_source' is deprecated; use 'django.template.loaders.filesystem.Loader' instead.",
+        PendingDeprecationWarning
+    )
+    return _loader.load_template_source(template_name, template_dirs)
 load_template_source.is_usable = True

django/test/utils.py

@@ -37,8 +37,8 @@ def setup_test_environment():
         - Set the email backend to the locmem email backend.
         - Setting the active locale to match the LANGUAGE_CODE setting.
     """
-    Template.original_render = Template.render
-    Template.render = instrumented_test_render
+    Template.original_render = Template._render
+    Template._render = instrumented_test_render
 
     mail.original_SMTPConnection = mail.SMTPConnection
     mail.SMTPConnection = locmem.EmailBackend
@@ -57,7 +57,7 @@ def teardown_test_environment():
         - Restoring the email sending functions
 
     """
-    Template.render = Template.original_render
+    Template._render = Template.original_render
     del Template.original_render
 
     mail.SMTPConnection = mail.original_SMTPConnection

django/views/debug.py

@@ -76,8 +76,12 @@ class ExceptionReporter:
                         for t in source_list_func(str(self.exc_value))]
                 except (ImportError, AttributeError):
                     template_list = []
+                if hasattr(loader, '__class__'):
+                    loader_name = loader.__module__ + '.' + loader.__class__.__name__
+                else:
+                    loader_name = loader.__module__ + '.' + loader.__name__
                 self.loader_debug_info.append({
-                    'loader': loader.__module__ + '.' + loader.__name__,
+                    'loader': loader_name,
                     'templates': template_list,
                 })
         if settings.TEMPLATE_DEBUG and hasattr(self.exc_value, 'source'):

docs/howto/custom-template-tags.txt

@@ -463,6 +463,85 @@ new ``Context`` in this example, the results would have *always* been
 automatically escaped, which may not be the desired behavior if the template
 tag is used inside a ``{% autoescape off %}`` block.
 
+.. _template_tag_thread_safety:
+
+Thread-safety considerations
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. versionadded:: 1.2
+
+Once a node is parsed, its ``render`` method may be called any number of times.
+Since Django is sometimes run in multi-threaded environments, a single node may
+be simultaneously rendering with different contexts in response to two separate
+requests. Therefore, it's important to make sure your template tags are thread
+safe.
+
+To make sure your template tags are thread safe, you should never store state
+information on the node itself. For example, Django provides a builtin ``cycle``
+template tag that cycles among a list of given strings each time it's rendered::
+
+    {% for o in some_list %}
+        <tr class="{% cycle 'row1' 'row2' %}>
+            ...
+        </tr>
+    {% endfor %}
+
+A naive implementation of ``CycleNode`` might look something like this::
+
+    class CycleNode(Node):
+        def __init__(self, cyclevars):
+            self.cycle_iter = itertools.cycle(cyclevars)
+        def render(self, context):
+            return self.cycle_iter.next()
+
+But, suppose we have two templates rendering the template snippet from above at
+the same time:
+
+    1. Thread 1 performs its first loop iteration, ``CycleNode.render()``
+       returns 'row1'
+    2. Thread 2 performs its first loop iteration, ``CycleNode.render()``
+       returns 'row2'
+    3. Thread 1 performs its second loop iteration, ``CycleNode.render()``
+       returns 'row1'
+    4. Thread 2 performs its second loop iteration, ``CycleNode.render()``
+       returns 'row2'
+
+The CycleNode is iterating, but it's iterating globally. As far as Thread 1
+and Thread 2 are concerned, it's always returning the same value. This is
+obviously not what we want!
+
+To address this problem, Django provides a ``render_context`` that's associated
+with the ``context`` of the template that is currently being rendered. The
+``render_context`` behaves like a Python dictionary, and should be used to store
+``Node`` state between invocations of the ``render`` method.
+
+Let's refactor our ``CycleNode`` implementation to use the ``render_context``::
+
+    class CycleNode(Node):
+        def __init__(self, cyclevars):
+            self.cyclevars = cyclevars
+        def render(self, context):
+            if self not in context.render_context:
+                context.render_context[self] = itertools.cycle(self.cyclevars)
+            cycle_iter = context.render_context[self]
+            return cycle_iter.next()
+
+Note that it's perfectly safe to store global information that will not change
+throughout the life of the ``Node`` as an attribute. In the case of
+``CycleNode``, the ``cyclevars`` argument doesn't change after the ``Node`` is
+instantiated, so we don't need to put it in the ``render_context``. But state
+information that is specific to the template that is currently being rendered,
+like the current iteration of the ``CycleNode``, should be stored in the
+``render_context``.
+
+.. note::
+    Notice how we used ``self`` to scope the ``CycleNode`` specific information
+    within the ``render_context``. There may be multiple ``CycleNodes`` in a
+    given template, so we need to be careful not to clobber another node's state
+    information. The easiest way to do this is to always use ``self`` as the key
+    into ``render_context``. If you're keeping track of several state variables,
+    make ``render_context[self]`` a dictionary.
+
 Registering the tag
 ~~~~~~~~~~~~~~~~~~~
 

docs/internals/deprecation.txt

@@ -36,14 +36,20 @@ their deprecation, as per the :ref:`Django deprecation policy
           manager in the ``User`` model (``user.message_set``), and the
           associated methods (``user.message_set.create()`` and
           ``user.get_and_delete_messages()``), which have
-          been deprecated since the 1.2 release, will be removed.  The 
-          :ref:`messages framework <ref-contrib-messages>` should be used 
+          been deprecated since the 1.2 release, will be removed.  The
+          :ref:`messages framework <ref-contrib-messages>` should be used
           instead.
 
         * Authentication backends need to support the ``obj`` parameter for
           permission checking. The ``supports_object_permissions`` variable
           is not checked any longer and can be removed.
 
+        * The ability to specify a callable template loader rather than a
+          ``Loader`` class will be removed, as will the ``load_template_source``
+          functions that are included with the built in template loaders for
+          backwards compatibility. These have been deprecated since the 1.2
+          release.
+
     * 2.0
         * ``django.views.defaults.shortcut()``. This function has been moved
           to ``django.contrib.contenttypes.views.shortcut()`` as part of the

docs/ref/contrib/sitemaps.txt

@@ -36,7 +36,7 @@ To install the sitemap app, follow these steps:
     1. Add ``'django.contrib.sitemaps'`` to your :setting:`INSTALLED_APPS`
        setting.
        
-    2. Make sure ``'django.template.loaders.app_directories.load_template_source'``
+    2. Make sure ``'django.template.loaders.app_directories.Loader'``
        is in your :setting:`TEMPLATE_LOADERS` setting. It's in there by default,
        so you'll only need to change this if you've changed that setting.
 
@@ -45,7 +45,7 @@ To install the sitemap app, follow these steps:
 
 (Note: The sitemap application doesn't install any database tables. The only
 reason it needs to go into :setting:`INSTALLED_APPS` is so that the
-:func:`~django.template.loaders.app_directories.load_template_source` template
+:func:`~django.template.loaders.app_directories.Loader` template
 loader can find the default templates.)
 
 Initialization

docs/ref/settings.txt

@@ -819,7 +819,7 @@ MESSAGE_LEVEL
 
 Default: `messages.INFO`
 
-Sets the minimum message level that will be recorded by the messages 
+Sets the minimum message level that will be recorded by the messages
 framework. See the :ref:`messages documentation <ref-contrib-messages>` for
 more details.
 
@@ -1150,11 +1150,14 @@ TEMPLATE_LOADERS
 
 Default::
 
-     ('django.template.loaders.filesystem.load_template_source',
-      'django.template.loaders.app_directories.load_template_source')
+     ('django.template.loaders.filesystem.Loader',
+      'django.template.loaders.app_directories.Loader')
 
-A tuple of callables (as strings) that know how to import templates from
-various sources. See :ref:`ref-templates-api`.
+A tuple of template loader classes, specified as strings. Each ``Loader`` class
+knows how to import templates from a particular sources. Optionally, a tuple can be
+used instead of a string. The first item in the tuple should be the ``Loader``'s
+module, subsequent items are passed to the ``Loader`` during initialization. See
+:ref:`ref-templates-api`.
 
 .. setting:: TEMPLATE_STRING_IF_INVALID
 

docs/ref/templates/api.txt

@@ -322,7 +322,7 @@ and return a dictionary of items to be merged into the context. By default,
    cannot be turned off by the :setting:`TEMPLATE_CONTEXT_PROCESSORS` setting.
 
 .. versionadded:: 1.2
-   The ``'messages'`` context processor was added.  For more information, see 
+   The ``'messages'`` context processor was added.  For more information, see
    the :ref:`messages documentation <ref-contrib-messages>`.
 
 Each processor is applied in order. That means, if one processor adds a
@@ -379,7 +379,7 @@ If :setting:`TEMPLATE_CONTEXT_PROCESSORS` contains this processor, every
 
 .. versionchanged:: 1.2
    Prior to version 1.2, the ``messages`` variable was a lazy accessor for
-   ``user.get_and_delete_messages()``. It has been changed to include any 
+   ``user.get_and_delete_messages()``. It has been changed to include any
    messages added via the :ref:`messages framework <ref-contrib-messages`.
 
 django.core.context_processors.debug
@@ -448,7 +448,7 @@ If :setting:`TEMPLATE_CONTEXT_PROCESSORS` contains this processor, every
    context processor.  For backwards compatibility the ``'auth'`` context
    processor will continue to supply the ``messages`` variable until Django
    1.4.  If you use the ``messages`` variable, your project will work with
-   either (or both) context processors, but it is recommended to add 
+   either (or both) context processors, but it is recommended to add
    ``django.contrib.messages.context_processors.messages`` so your project
    will be prepared for the future upgrade.
 
@@ -571,11 +571,11 @@ by editing your :setting:`TEMPLATE_LOADERS` setting. :setting:`TEMPLATE_LOADERS`
 should be a tuple of strings, where each string represents a template loader.
 Here are the template loaders that come with Django:
 
-``django.template.loaders.filesystem.load_template_source``
+``django.template.loaders.filesystem.Loader``
     Loads templates from the filesystem, according to :setting:`TEMPLATE_DIRS`.
     This loader is enabled by default.
 
-``django.template.loaders.app_directories.load_template_source``
+``django.template.loaders.app_directories.Loader``
     Loads templates from Django apps on the filesystem. For each app in
     :setting:`INSTALLED_APPS`, the loader looks for a ``templates``
     subdirectory. If the directory exists, Django looks for templates in there.
@@ -599,12 +599,43 @@ Here are the template loaders that come with Django:
 
     This loader is enabled by default.
 
-``django.template.loaders.eggs.load_template_source``
+``django.template.loaders.eggs.Loader``
     Just like ``app_directories`` above, but it loads templates from Python
     eggs rather than from the filesystem.
 
     This loader is disabled by default.
 
+``django.template.loaders.cached.Loader``
+    By default, the templating system will read and compile your templates every
+    time they need to be rendered. While the Django templating system is quite
+    fast, the overhead from reading and compiling templates can add up.
+
+    The cached template loader is a class-based loader that you configure with
+    a list of other loaders that it should wrap. The wrapped loaders are used to
+    locate unknown templates when they are first encountered. The cached loader
+    then stores the compiled ``Template`` in memory. The cached ``Template``
+    instance is returned for subsequent requests to load the same template.
+
+    For example, to enable template caching with the ``filesystem`` and
+    ``app_directories`` template loaders you might use the following settings::
+
+        TEMPLATE_LOADERS = (
+            ('django.template.loaders.cached.Loader', (
+                'django.template.loaders.filesystem.Loader',
+                'django.template.loaders.app_directories.Loader',
+            )),
+        )
+
+    .. note::
+        All of the built-in Django template tags are safe to use with the cached
+        loader, but if you're using custom template tags that come from third
+        party packages, or that you wrote yourself, you should ensure that the
+        ``Node`` implementation for each tag is thread-safe. For more
+        information, see
+        :ref:`template tag thread safety considerations<template_tag_thread_safety>`.
+
+    This loader is disabled by default.
+
 Django uses the template loaders in order according to the
 :setting:`TEMPLATE_LOADERS` setting. It uses each loader until a loader finds a
 match.
@@ -667,3 +698,68 @@ settings you wish to specify. You might want to consider setting at least
 and :setting:`TEMPLATE_DEBUG`. All available settings are described in the
 :ref:`settings documentation <ref-settings>`, and any setting starting with
 ``TEMPLATE_`` is of obvious interest.
+
+.. _topic-template-alternate-language:
+
+Using an alternative template language
+======================================
+
+.. versionadded 1.2
+
+The Django ``Template`` and ``Loader`` classes implement a simple API for
+loading and rendering templates. By providing some simple wrapper classes that
+implement this API we can use third party template systems like `Jinja2
+<http://jinja.pocoo.org/2/>`_ or `Cheetah <http://www.cheetahtemplate.org/>`_. This
+allows us to use third-party template libraries without giving up useful Django
+features like the Django ``Context`` object and handy shortcuts like
+``render_to_response()``.
+
+The core component of the Django templating system is the ``Template`` class.
+This class has a very simple interface: it has a constructor that takes a single
+positional argument specifying the template string, and a ``render()`` method
+that takes a ``django.template.context.Context`` object and returns a string
+containing the rendered response.
+
+Suppose we're using a template language that defines a ``Template`` object with
+a ``render()`` method that takes a dictionary rather than a ``Context`` object.
+We can write a simple wrapper that implements the Django ``Template`` interface::
+
+    import some_template_language
+    class Template(some_template_language.Template):
+        def render(self, context):
+            # flatten the Django Context into a single dictionary.
+            context_dict = {}
+            for d in context.dicts:
+                context_dict.update(d)
+            return super(Template, self).render(context_dict)
+
+That's all that's required to make our fictional ``Template`` class compatible
+with the Django loading and rendering system!
+
+The next step is to write a ``Loader`` class that returns instances of our custom
+template class instead of the default ``django.template.Template``. Custom ``Loader``
+classes should inherit from ``django.template.loader.BaseLoader`` and override
+the ``load_template_source()`` method, which takes a ``template_name`` argument,
+loads the template from disk (or elsewhere), and returns a tuple:
+``(template_string, template_origin)``.
+
+The ``load_template()`` method of the ``Loader`` class retrieves the template
+string by calling ``load_template_source()``, instantiates a ``Template`` from
+the template source, and returns a tuple: ``(template, template_origin)``. Since
+this is the method that actually instantiates the ``Template``, we'll need to
+override it to use our custom template class instead. We can inherit from the
+builtin ``django.template.loaders.app_directories.Loader`` to take advantage of
+the ``load_template_source()`` method implemented there::
+
+    from django.template.loaders import app_directories
+    class Loader(app_directories.Loader):
+        is_usable = True
+
+        def load_template(self, template_name, template_dirs=None):
+            source, origin = self.load_template_source(template_name, template_dirs)
+            template = Template(source)
+            return template, origin
+
+Finally, we need to modify our project settings, telling Django to use our custom
+loader. Now we can write all of our templates in our alternative template
+language while continuing to use the rest of the Django templating system.

docs/releases/1.2.txt

@@ -76,6 +76,19 @@ changes:
 
        __members__ = property(lambda self: self.__dir__())
 
+Stateful template tags
+----------------------
+
+Template tags that store rendering state on the node itself may experience
+problems if they are used with the new :ref:`cached
+template loader<template-loaders>`.
+
+All of the built-in Django template tags are safe to use with the cached
+loader, but if you're using custom template tags that come from third
+party packages, or that you wrote yourself, you should ensure that the
+``Node`` implementation for each tag is thread-safe. For more
+information, see
+:ref:`template tag thread safety considerations<template_tag_thread_safety>`.
 
 .. _deprecated-features-1.2:
 
@@ -130,11 +143,11 @@ additional arguments, those arguments can be passed to the
 :meth:`~django.core.mail.get_connection()` call::
 
     connection = get_connection('django.core.mail.backends.smtp', hostname='localhost', port=1234)
-    
+
 User Messages API
 -----------------
 
-The API for storing messages in the user ``Message`` model (via 
+The API for storing messages in the user ``Message`` model (via
 ``user.message_set.create``) is now deprecated and will be removed in Django
 1.4 according to the standard :ref:`release process <internals-release-process>`.
 
@@ -147,20 +160,20 @@ with the following::
     from django.contrib import messages
     messages.add_message(request, messages.INFO, 'a message')
 
-Additionally, if you make use of the method, you need to replace the 
+Additionally, if you make use of the method, you need to replace the
 following::
 
     for message in user.get_and_delete_messages():
         ...
-    
+
 with::
 
     from django.contrib import messages
     for message in messages.get_messages(request):
         ...
-    
-For more information, see the full 
-:ref:`messages documentation <ref-contrib-messages>`. You should begin to 
+
+For more information, see the full
+:ref:`messages documentation <ref-contrib-messages>`. You should begin to
 update your code to use the new API immediately.
 
 What's new in Django 1.2
@@ -239,3 +252,18 @@ Also, filters may now be used in the ``if`` expression. For example:
           class="highlight"
         {% endif %}
       >{{ message }}</div>
+
+Template caching
+----------------
+
+In previous versions of Django, every time you rendered a template it
+would be reloaded from disk. In Django 1.2, you can use a :ref:`cached
+template loader <template-loaders>` to load templates once, then use a
+cached the result for every subsequent render. This can lead to a
+significant performance improvement if your templates are broken into
+lots of smaller subtemplates (using the ``{% extends %}`` or ``{%
+include %}`` tags).
+
+As a side effect, it is now much easier to support non-Django template
+languages. For more details, see the :ref:`notes on supporting
+non-Django template languages<topic-template-alternate-language>`.

tests/regressiontests/templates/context.py

@@ -10,9 +10,13 @@ context_tests = r"""
 >>> c['a'] = 2
 >>> c['a']
 2
+>>> c.get('a')
+2
 >>> c.pop()
 {'a': 2}
 >>> c['a']
 1
+>>> c.get('foo', 42)
+42
 """
 

tests/regressiontests/templates/tests.py

@@ -15,7 +15,7 @@ import unittest
 from django import template
 from django.core import urlresolvers
 from django.template import loader
-from django.template.loaders import app_directories, filesystem
+from django.template.loaders import app_directories, filesystem, cached
 from django.utils.translation import activate, deactivate, ugettext as _
 from django.utils.safestring import mark_safe
 from django.utils.tzinfo import LocalTimezone
@@ -101,13 +101,15 @@ class UTF8Class:
 
 class Templates(unittest.TestCase):
     def test_loaders_security(self):
+        ad_loader = app_directories.Loader()
+        fs_loader = filesystem.Loader()
         def test_template_sources(path, template_dirs, expected_sources):
             if isinstance(expected_sources, list):
                 # Fix expected sources so they are normcased and abspathed
                 expected_sources = [os.path.normcase(os.path.abspath(s)) for s in expected_sources]
             # Test the two loaders (app_directores and filesystem).
-            func1 = lambda p, t: list(app_directories.get_template_sources(p, t))
-            func2 = lambda p, t: list(filesystem.get_template_sources(p, t))
+            func1 = lambda p, t: list(ad_loader.get_template_sources(p, t))
+            func2 = lambda p, t: list(fs_loader.get_template_sources(p, t))
             for func in (func1, func2):
                 if isinstance(expected_sources, list):
                     self.assertEqual(func(path, template_dirs), expected_sources)
@@ -198,8 +200,11 @@ class Templates(unittest.TestCase):
             except KeyError:
                 raise template.TemplateDoesNotExist, template_name
 
+        cache_loader = cached.Loader(('test_template_loader',))
+        cache_loader._cached_loaders = (test_template_loader,)
+
         old_template_loaders = loader.template_source_loaders
-        loader.template_source_loaders = [test_template_loader]
+        loader.template_source_loaders = [cache_loader]
 
         failures = []
         tests = template_tests.items()
@@ -232,20 +237,22 @@ class Templates(unittest.TestCase):
             for invalid_str, result in [('', normal_string_result),
                                         (expected_invalid_str, invalid_string_result)]:
                 settings.TEMPLATE_STRING_IF_INVALID = invalid_str
-                try:
-                    test_template = loader.get_template(name)
-                    output = self.render(test_template, vals)
-                except ContextStackException:
-                    failures.append("Template test (TEMPLATE_STRING_IF_INVALID='%s'): %s -- FAILED. Context stack was left imbalanced" % (invalid_str, name))
-                    continue
-                except Exception:
-                    exc_type, exc_value, exc_tb = sys.exc_info()
-                    if exc_type != result:
-                        tb = '\n'.join(traceback.format_exception(exc_type, exc_value, exc_tb))
-                        failures.append("Template test (TEMPLATE_STRING_IF_INVALID='%s'): %s -- FAILED. Got %s, exception: %s\n%s" % (invalid_str, name, exc_type, exc_value, tb))
-                    continue
-                if output != result:
-                    failures.append("Template test (TEMPLATE_STRING_IF_INVALID='%s'): %s -- FAILED. Expected %r, got %r" % (invalid_str, name, result, output))
+                for is_cached in (False, True):
+                    try:
+                        test_template = loader.get_template(name)
+                        output = self.render(test_template, vals)
+                    except ContextStackException:
+                        failures.append("Template test (Cached='%s', TEMPLATE_STRING_IF_INVALID='%s'): %s -- FAILED. Context stack was left imbalanced" % (is_cached, invalid_str, name))
+                        continue
+                    except Exception:
+                        exc_type, exc_value, exc_tb = sys.exc_info()
+                        if exc_type != result:
+                            tb = '\n'.join(traceback.format_exception(exc_type, exc_value, exc_tb))
+                            failures.append("Template test (Cached='%s', TEMPLATE_STRING_IF_INVALID='%s'): %s -- FAILED. Got %s, exception: %s\n%s" % (is_cached, invalid_str, name, exc_type, exc_value, tb))
+                        continue
+                    if output != result:
+                        failures.append("Template test (Cached='%s', TEMPLATE_STRING_IF_INVALID='%s'): %s -- FAILED. Expected %r, got %r" % (is_cached, invalid_str, name, result, output))
+                cache_loader.reset()
 
             if 'LANGUAGE_CODE' in vals[1]:
                 deactivate()

tests/regressiontests/test_client_regress/models.py

@@ -10,6 +10,7 @@ from django.test.utils import ContextList
 from django.core.urlresolvers import reverse
 from django.core.exceptions import SuspiciousOperation
 from django.template import TemplateDoesNotExist, TemplateSyntaxError, Context
+from django.template import loader
 
 class AssertContainsTests(TestCase):
     def setUp(self):
@@ -436,6 +437,11 @@ class ExceptionTests(TestCase):
 
 class TemplateExceptionTests(TestCase):
     def setUp(self):
+        # Reset the loaders so they don't try to render cached templates.
+        if loader.template_source_loaders is not None:
+            for template_loader in loader.template_source_loaders:
+                if hasattr(template_loader, 'reset'):
+                    template_loader.reset()
         self.old_templates = settings.TEMPLATE_DIRS
         settings.TEMPLATE_DIRS = ()