Fixed tests, added support for import extend & include

cratejoy · aelaguiz · Dec 25, 2014 · Dec 26, 2014 · Dec 26, 2014 · Dec 26, 2014
commit 2ffcf5c1a5705fc761ff5d8f25a689fe19923801
diff --git a/html5lib/constants.py b/html5lib/constants.py
@@ -3096,7 +3096,10 @@
     "JinjaPipe": 15,
     "JinjaArgumentStartTag": 16,
     "JinjaArgumentEndTag": 17,
-    "JinjaArgument": 18
+    "JinjaArgument": 18,
+    "JinjaExtendTag": 19,
+    "JinjaIncludeTag": 20,
+    "JinjaImportTag": 21
 }
 
 tagTokenTypes = frozenset((tokenTypes["StartTag"], tokenTypes["EndTag"],

diff --git a/html5lib/html5parser.py b/html5lib/html5parser.py
@@ -177,6 +177,9 @@ def mainLoop(self):
         JinjaArgumentStartTag = tokenTypes["JinjaArgumentStartTag"]
         JinjaArgumentEndTag = tokenTypes["JinjaArgumentEndTag"]
         JinjaArgument = tokenTypes["JinjaArgument"]
+        JinjaExtendTag = tokenTypes["JinjaExtendTag"]
+        JinjaIncludeTag = tokenTypes["JinjaIncludeTag"]
+        JinjaImportTag = tokenTypes["JinjaImportTag"]
 
         for token in self.normalizedTokens():
             new_token = token
@@ -193,13 +196,10 @@ def mainLoop(self):
                     new_token = None
                 else:
                     if type in (JinjaVariableStartTag, JinjaVariableEndTag, JinjaVariable, JinjaFilter, JinjaPipe):
-                        log.debug(u"Type is a jinja variable tag")
                         phase = self.phases["inJinjaVariable"]
                     elif type in (JinjaStatementStartTag, JinjaStatementEndTag, JinjaStatement):
-                        log.debug(u"Type is a jinja statement tag")
                         phase = self.phases["inJinjaStatement"]
                     elif type in (JinjaArgumentStartTag, JinjaArgumentEndTag, JinjaArgument):
-                        log.debug(u"Type is a jinja argument tag")
                         phase = self.phases["inJinjaArgument"]
                     elif (
                         len(self.tree.openElements) == 0 or
@@ -251,6 +251,12 @@ def mainLoop(self):
                         new_token = phase.processJinjaArgumentEndTag(new_token)
                     elif type == JinjaArgument:
                         new_token = phase.processJinjaArgument(new_token)
+                    elif type == JinjaExtendTag:
+                        new_token = phase.processJinjaExtendTag(new_token)
+                    elif type == JinjaIncludeTag:
+                        new_token = phase.processJinjaIncludeTag(new_token)
+                    elif type == JinjaImportTag:
+                        new_token = phase.processJinjaImportTag(new_token)
 
             if (type == StartTagToken and token["selfClosing"]
                     and not token["selfClosingAcknowledged"]):
@@ -546,6 +552,18 @@ def processJinjaVariableEndTag(self, token):
         def processJinjaVariable(self, token):
             pass
 
+        def processJinjaExtendTag(self, token):
+            element = self.tree.createElementWithoutNamespace(token)
+            self.tree.openElements[-1].appendChild(element)
+
+        def processJinjaIncludeTag(self, token):
+            element = self.tree.createElementWithoutNamespace(token)
+            self.tree.openElements[-1].appendChild(element)
+
+        def processJinjaImportTag(self, token):
+            element = self.tree.createElementWithoutNamespace(token)
+            self.tree.openElements[-1].appendChild(element)
+
         def processJinjaArgumentStartTag(self, token):
             pass
 

diff --git a/html5lib/tests/test_jinja.py b/html5lib/tests/test_jinja.py
@@ -15,7 +15,7 @@ def dump(tree, tabs=0):
 
 class JinjaTestCase(unittest.TestCase):
     def setUp(self):
-        self.parser = html5lib.HTMLParser(strict=True, namespaceHTMLElements=False)
+        self.parser = html5lib.HTMLParser(strict=True, namespaceHTMLElements=False, tree=html5lib.treebuilders.getTreeBuilder("etree", fullTree=True))
 
     def test_var_1(self):
         html_string = """<h1>{{ hi }}</h1>"""
@@ -197,42 +197,48 @@ def test_complete_doc(self):
         """
 
         tree = self.parser.parse(html_string)
-        dump(tree)
+
         self.assertTree(tree, [{
-            'tag': 'head',
-            'children': [{
-                'tag': 'title',
-                'text': 'My Webpage'
-            }]
+            'tag': '<!DOCTYPE>',
+            'text': 'html'
         }, {
-            'tag': 'body',
+            'tag': 'html',
             'children': [{
-                'tag': 'ul',
+                'tag': 'head',
                 'children': [{
-                    'tag': 'jinjafor',
-                    'value': 'item in navigation',
+                    'tag': 'title',
+                    'text': 'My Webpage'
+                }]
+            }, {
+                'tag': 'body',
+                'children': [{
+                    'tag': 'ul',
                     'children': [{
-                        'tag': 'li',
+                        'tag': 'jinjafor',
+                        'value': 'item in navigation',
                         'children': [{
-                            'tag': 'a',
+                            'tag': 'li',
                             'children': [{
-                                'tag': 'jinjavariabletag',
+                                'tag': 'a',
                                 'children': [{
-                                    'tag': 'jinjavariable',
-                                    'value': 'item.caption'
+                                    'tag': 'jinjavariabletag',
+                                    'children': [{
+                                        'tag': 'jinjavariable',
+                                        'value': 'item.caption'
+                                    }]
                                 }]
                             }]
                         }]
                     }]
-                }]
-            }, {
-                'tag': 'h1',
-                'text': 'My Webpage'
-            }, {
-                'tag': 'jinjavariabletag',
-                'children': [{
-                    'tag': 'jinjavariable',
-                    'value': 'a_variable'
+                }, {
+                    'tag': 'h1',
+                    'text': 'My Webpage'
+                }, {
+                    'tag': 'jinjavariabletag',
+                    'children': [{
+                        'tag': 'jinjavariable',
+                        'value': 'a_variable'
+                    }]
                 }]
             }]
         }])
@@ -250,6 +256,89 @@ def test_jinja_if(self):
             'text': 'yay'
         }])
 
+    def test_jinja_if_lstrip(self):
+        html_string = """
+            {%+ if True %}yay{% endif %}
+        """
+
+        tree = self.parser.parseFragment(html_string)
+        dump(tree)
+
+        self.assertTree(tree, [{
+            'tag': 'jinjaif',
+            'text': 'yay',
+            'attrs': {
+                'lstrip': False
+            }
+        }])
+
+    def test_jinja_strip_blocks(self):
+        html_string = """
+            {% for item in seq -%}
+                {{ item }}
+            {%- endfor %}
+        """
+
+        tree = self.parser.parseFragment(html_string)
+        dump(tree)
+
+        self.assertTree(tree, [{
+            'tag': 'jinjafor',
+            'attrs': {
+                'rstrip': True
+            },
+            'children': [{
+                'tag': 'jinjavariabletag',
+                'children': [{
+                    'tag': 'jinjavariable',
+                    'value': 'item'
+                }]
+            }]
+        }])
+
+    def test_jinja_extend(self):
+        html_string = """
+            {% extends "base.html" %}
+        """
+
+        tree = self.parser.parseFragment(html_string)
+        dump(tree)
+
+        self.assertTree(tree, [{
+            'tag': 'jinjaextends',
+            'value': '"base.html"'
+        }])
+
+    def test_jinja_include(self):
+        html_string = """
+            {% include ['special_sidebar.html', 'sidebar.html'] ignore missing %}
+        """
+
+        tree = self.parser.parseFragment(html_string)
+        dump(tree)
+
+        self.assertTree(tree, [{
+            'tag': 'jinjainclude',
+            'value': "['special_sidebar.html', 'sidebar.html'] ignore missing"
+        }])
+
+    def test_jinja_import(self):
+        html_string = """
+            {% import 'forms.html' as forms %}
+            {% from 'forms.html' import input as input_field, textarea %}
+        """
+
+        tree = self.parser.parseFragment(html_string)
+        dump(tree)
+
+        self.assertTree(tree, [{
+            'tag': 'jinjaimport',
+            'value': "'forms.html' as forms"
+        }, {
+            'tag': 'jinjaimport',
+            'value': "'forms.html' import input as input_field, textarea"
+        }])
+
     def assertTree(self, root, spec):
         self.assertEqual(len(root), len(spec))
 
@@ -264,3 +353,10 @@ def assertTree(self, root, spec):
 
             if 'children' in spec_child:
                 self.assertTree(child, spec_child['children'])
+            else:
+                self.assertEqual(len(child), 0)
+
+            if 'attrs' in spec_child:
+                for k, v in spec_child['attrs'].iteritems():
+                    self.assertIn(k, child.attrib)
+                    self.assertEqual(v, child.attrib[k])
diff --git a/html5lib/tests/test_parser2.py b/html5lib/tests/test_parser2.py
@@ -40,7 +40,7 @@ def test_namespace_html_elements_1_dom(self):
     def test_namespace_html_elements_0_etree(self):
         parser = html5parser.HTMLParser(namespaceHTMLElements=True)
         doc = parser.parse("<html></html>")
-        self.assertTrue(list(doc)[0].tag == "{%s}html" % (namespaces["html"],))
+        self.assertEqual(list(doc)[0].tag, "{%s}html" % (namespaces["html"],))
 
     def test_namespace_html_elements_1_etree(self):
         parser = html5parser.HTMLParser(namespaceHTMLElements=False)

diff --git a/html5lib/tokenizer.py b/html5lib/tokenizer.py
@@ -328,31 +328,66 @@ def jinjaStatementStartState(self):
                                     "eof-in-jinja-statement"})
             self.state = self.prevState
         else:
+            attrs = {}
+
+            if data in ['-', '+']:
+                attrs['lstrip'] = False
+
+                data = self.stream.char()
+                while data in spaceCharacters:
+                    data = self.stream.char()
+
             block_type = data + self.stream.charsUntil(frozenset(("%")) | spaceCharacters)
 
             block_definition = self.stream.charsUntil(frozenset(("%", "\u0000")))
 
             block_definition = block_definition.strip(" \t")
 
+            if block_definition and block_definition[-1] == '-':
+                attrs['rstrip'] = True
+                block_definition = block_definition[:-1].rstrip()
+
+            attrs.update({
+                "value": block_definition,
+                "position": self.stream.position()
+            })
+
             if block_type.startswith("end"):
                 block_type = block_type.replace("end", "")
                attrs['value'] = block_type.lower()
 
                 self.tokenQueue.append({
                     "type": tokenTypes["JinjaStatementEndTag"],
                     'name': u"jinja{}".format(block_type.lower()),
-                    "data": {
-                        "position": self.stream.position()
-                    },
+                    "data": attrs,
                     "selfClosing": False
                 })
+            elif block_type == "extends":
+                self.tokenQueue.append({
+                    "type": tokenTypes["JinjaExtendTag"],
+                    'name': u"jinja{}".format(block_type.lower()),
+                    "data": attrs,
+                    "selfClosing": True
+                })
+            elif block_type == "include":
+                self.tokenQueue.append({
+                    "type": tokenTypes["JinjaIncludeTag"],
+                    'name': u"jinja{}".format(block_type.lower()),
+                    "data": attrs,
+                    "selfClosing": True
+                })
+            elif block_type in ["import", "from"]:
+                self.tokenQueue.append({
+                    "type": tokenTypes["JinjaImportTag"],
+                    'name': u"jinjaimport",
+                    "data": attrs,
+                    "selfClosing": True
+                })
             else:
                 self.tokenQueue.append({
                     "type": tokenTypes["JinjaStatementStartTag"],
                     'name': u"jinja{}".format(block_type.lower()),
-                    "data": {
-                        "value": block_definition,
-                        "position": self.stream.position()
-                    },
+                    "data": attrs,
                     "selfClosing": False
                 })
 

diff --git a/html5lib/treebuilders/etree.py b/html5lib/treebuilders/etree.py
@@ -63,7 +63,6 @@ def _getAttributes(self):
             return self._element.attrib
 
         def _setAttributes(self, attributes):
-            log.debug(u"Attributes {}".format(attributes))
             # Delete existing attributes first
             # XXX - there may be a better way to do this...
             for key in list(self._element.attrib.keys()):