html5lib
diff --git a/‎html5lib/constants.py
Lines changed: 2 additions & 1 deletion b/‎html5lib/constants.py
Lines changed: 2 additions & 1 deletion
diff --git a/‎html5lib/html5parser.py
Lines changed: 50 additions & 5 deletions b/‎html5lib/html5parser.py
Lines changed: 50 additions & 5 deletions
diff --git a/‎html5lib/tokenizer.py
Lines changed: 36 additions & 8 deletions b/‎html5lib/tokenizer.py
Lines changed: 36 additions & 8 deletions
diff --git a/‎html5lib/treebuilders/_base.py
Lines changed: 12 additions & 0 deletions b/‎html5lib/treebuilders/_base.py
Lines changed: 12 additions & 0 deletions
diff --git a/‎html5lib/treebuilders/etree.py
Lines changed: 5 additions & 0 deletions b/‎html5lib/treebuilders/etree.py
Lines changed: 5 additions & 0 deletions
@@ -3091,7 +3091,8 @@
     "JinjaStatementTag": 10,
     "JinjaVariableStartTag": 11,
     "JinjaVariableEndTag": 12,
-    "JinjaVariableTag": 13
+    "JinjaVariable": 13,
+    "JinjaFilter": 14
 }
 
 tagTokenTypes = frozenset((tokenTypes["StartTag"], tokenTypes["EndTag"],
 
@@ -171,7 +171,8 @@ def mainLoop(self):
         JinjaStatementTag = tokenTypes["JinjaStatementTag"]
         JinjaVariableStartTag = tokenTypes["JinjaVariableStartTag"]
         JinjaVariableEndTag = tokenTypes["JinjaVariableEndTag"]
-        JinjaVariableTag = tokenTypes["JinjaVariableTag"]
+        JinjaVariable = tokenTypes["JinjaVariable"]
+        JinjaFilter = tokenTypes["JinjaFilter"]
 
         for token in self.normalizedTokens():
             new_token = token
@@ -187,7 +188,11 @@ def mainLoop(self):
                     self.parseError(new_token["data"], new_token.get("datavars", {}))
                     new_token = None
                 else:
-                    if (len(self.tree.openElements) == 0 or
+                    if type in (JinjaVariableStartTag, JinjaVariableEndTag, JinjaVariable, JinjaFilter):
+                        log.debug(u"Type is a jinja tag")
+                        phase = self.phases["inJinjaVariable"]
+                    elif (
+                        len(self.tree.openElements) == 0 or
                         currentNodeNamespace == self.tree.defaultNamespace or
                         (self.isMathMLTextIntegrationPoint(currentNode) and
                          ((type == StartTagToken and
@@ -224,8 +229,10 @@ def mainLoop(self):
                         new_token = phase.processJinjaVariableStartTag(new_token)
                     elif type == JinjaVariableEndTag:
                         new_token = phase.processJinjaVariableEndTag(new_token)
-                    elif type == JinjaVariableTag:
-                        new_token = phase.processJinjaVariableTag(new_token)
+                    elif type == JinjaVariable:
+                        new_token = phase.processJinjaVariable(new_token)
+                    elif type == JinjaFilter:
+                        new_token = phase.processJinjaFilter(new_token)
 
             if (type == StartTagToken and token["selfClosing"]
                     and not token["selfClosingAcknowledged"]):
@@ -516,7 +523,10 @@ def processJinjaVariableStartTag(self, token):
         def processJinjaVariableEndTag(self, token):
             pass
 
-        def processJinjaVariableTag(self, token):
+        def processJinjaVariable(self, token):
+            pass
+
+        def processJinjaFilterTag(self, token):
             pass
 
         def processStartTag(self, token):
@@ -535,6 +545,40 @@ def startTagHtml(self, token):
         def processEndTag(self, token):
             return self.endTagHandler[token["name"]](token)
 
+    class InJinjaVariablePhase(Phase):
+        def processJinjaVariableStartTag(self, token):
+            log = logging.getLogger('html5lib')
+            log.debug(u"InJinja: Start Tag")
+            self.tree.reconstructActiveFormattingElements()
+            self.tree.insertElement(token)
+
+        def processJinjaVariableEndTag(self, token):
+            log = logging.getLogger('html5lib')
+            log.debug(u"InJinja: End Tag {}".format(token["name"]))
+            for node in self.tree.openElements[::-1]:
+                log.debug(u"InJinja: Open tag {} token {}".format(node, token))
+                if node.name == token["name"]:
+                    self.tree.generateImpliedEndTags(exclude=token["name"])
+                    log.debug(u"InJinja: Implied end tag {} {}".format(self.tree.openElements[-1].name, token["name"]))
+                    if self.tree.openElements[-1].name != token["name"]:
+                        self.parser.parseError("unexpected-end-tag", {"name": token["name"]})
+                    while self.tree.openElements.pop() != node:
+                        pass
+                    break
+                else:
+                    if node.nameTuple in specialElements:
+                        log.debug(u"Nametuple {} in {}".format(node.nameTuple, specialElements))
+                        self.parser.parseError("unexpected-end-tag", {"name": token["name"]})
+                        break
+
+        def processJinjaVariable(self, token):
+            element = self.tree.createElementWithoutNamespace(token)
+            self.tree.openElements[-1].appendChild(element)
+
+        def processJinjaFilter(self, token):
+            element = self.tree.createElementWithoutNamespace(token)
+            self.tree.openElements[-1].appendChild(element)
+
     class InitialPhase(Phase):
         def processSpaceCharacters(self, token):
             pass
@@ -2739,6 +2783,7 @@ def processEndTag(self, token):
         "inHead": InHeadPhase,
         # XXX "inHeadNoscript": InHeadNoScriptPhase,
         "afterHead": AfterHeadPhase,
+        "inJinjaVariable": InJinjaVariablePhase,
         "inBody": InBodyPhase,
         "text": TextPhase,
         "inTable": InTablePhase,
 
@@ -294,17 +294,21 @@ def jinjaOpenState(self):
         data = self.stream.char()
 
         if data == "{":
-            self.tokenQueue.append({
+            self.currentToken = {
                 "type": tokenTypes["JinjaVariableStartTag"],
-                "name": "{{", "data": [],
+                "name": u"jinjavariabletag", "data": {},
+                "namespace": None,
                 "selfClosing": False
-            })
+            }
+
+            self.tokenQueue.append(self.currentToken)
 
             self.state = self.jinjaVariableState
         elif data == "%":
             self.tokenQueue.append({
                 "type": tokenTypes["JinjaStatementStartTag"],
-                "name": "{%", "data": [],
+                "name": "{%", "data": {},
+                "namespace": None,
                 "selfClosing": False
             })
 
@@ -346,7 +350,7 @@ def jinjaVariableEndState(self):
         if data == "}":
             self.tokenQueue.append({
                 "type": tokenTypes["JinjaVariableEndTag"],
-                "name": "}}", "data": [],
+                "name": u"jinjavariabletag", "data": [],
                 "selfClosing": False
             })
             self.state = self.dataState
@@ -390,10 +394,34 @@ def jinjaVariableState(self):
             self.tokenQueue.append({"type": tokenTypes["ParseError"], "data":
                                     "eof-in-jinja-variable"})
             self.state = self.dataState
+        elif data in spaceCharacters:
+            # Skip spaces
+            pass
+        elif data == "|":
+            pass
+        # If this is the first token after the variable start tag
+        elif self.currentToken['type'] == tokenTypes["JinjaVariableStartTag"]:
+            #log.debug(u"Got start tag {}".format(("|", "}", "\u0000") | spaceCharacters))
+
+            chars = self.stream.charsUntil(frozenset(("|", "}", "\u0000")) | spaceCharacters)
+            self.currentToken = {"type": tokenTypes["JinjaVariable"], 
+                                    "name": "jinjavariable", "selfClosing": True, "data": { 
+                                        "value": data + chars,
+                                        "position": self.stream.position(),
+                                    }}
+            self.tokenQueue.append(self.currentToken)
         else:
-            chars = self.stream.charsUntil(("}", "\u0000"))
-            self.tokenQueue.append({"type": tokenTypes["JinjaVariableTag"], "data":
-                                    data + chars})
+            chars = self.stream.charsUntil(frozenset(("|", "}", "\u0000")) | spaceCharacters)
+            self.currentToken = {"type": tokenTypes["JinjaFilter"], 
+                                    "name": "jinjafilter", "selfClosing": True, "data": { 
+                                        "value": data + chars,
+                                        "position": self.stream.position(),
+                                    }}
+            self.tokenQueue.append(self.currentToken)
+        #else:
+            #chars = self.stream.charsUntil(("}", "\u0000"))
+            #self.tokenQueue.append({"type": tokenTypes["JinjaVariableTag"], "data":
+                                    #data + chars})
 
         return True
 
 
@@ -3,6 +3,11 @@
 
 from ..constants import scopingElements, tableInsertModeElements, namespaces
 
+import logging
+
+log = logging.getLogger("html5lib")
+
+
 # The scope markers are inserted when entering object elements,
 # marquees, table cells, and table captions, and are used to prevent formatting
 # from "leaking" into tables, object elements, and marquees.
@@ -269,6 +274,13 @@ def createElement(self, token):
         element.attributes = token["data"]
         return element
 
+    def createElementWithoutNamespace(self, token):
+        """Create an element but don't insert it anywhere"""
+        name = token["name"]
+        element = self.elementClass(name)
+        element.attributes = token["data"]
+        return element
+
     def _getInsertFromTable(self):
         return self._insertFromTable
 
 
@@ -3,6 +3,8 @@
 
 import re
 
+import logging
+
 from . import _base
 from .. import ihatexml
 from .. import constants
@@ -11,6 +13,8 @@
 
 tag_regexp = re.compile("{([^}]*)}(.*)")
 
+log = logging.getLogger("html5lib")
+
 
 def getETreeBuilder(ElementTreeImplementation, fullTree=False):
     ElementTree = ElementTreeImplementation
@@ -59,6 +63,7 @@ def _getAttributes(self):
             return self._element.attrib
 
         def _setAttributes(self, attributes):
+            log.debug(u"Attributes {}".format(attributes))
             # Delete existing attributes first
             # XXX - there may be a better way to do this...
             for key in list(self._element.attrib.keys()):