8000 Fixed tests, added support for import extend & include · cratejoy/html5lib-python@2ffcf5c · GitHub
[go: up one dir, main page]
Skip to content

Commit 2ffcf5c

Browse files
aelaguizaelaguiz
committed
Fixed tests, added support for import extend & include
1 parent c12b08f commit 2ffcf5c

File tree

6 files changed

+189
-38
lines changed

6 files changed

+189
-38
lines changed

‎html5lib/constants.py

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3096,7 +3096,10 @@
30963096
"JinjaPipe": 15,
30973097
"JinjaArgumentStartTag": 16,
30983098
"JinjaArgumentEndTag": 17,
3099-
"JinjaArgument": 18
3099+
"JinjaArgument": 18,
3100+
"JinjaExtendTag": 19,
3101+
"JinjaIncludeTag": 20,
3102+
"JinjaImportTag": 21
31003103
}
31013104

31023105
tagTokenTypes = frozenset((tokenTypes["StartTag"], tokenTypes["EndTag"],

‎html5lib/html5parser.py

Lines changed: 21 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -177,6 +177,9 @@ def mainLoop(self):
177177
JinjaArgumentStartTag = tokenTypes["JinjaArgumentStartTag"]
178178
JinjaArgumentEndTag = tokenTypes["JinjaArgumentEndTag"]
179179
JinjaArgument = tokenTypes["JinjaArgument"]
180+
JinjaExtendTag = tokenTypes["JinjaExtendTag"]
181+
JinjaIncludeTag = tokenTypes["JinjaIncludeTag"]
182+
JinjaImportTag = tokenTypes["JinjaImportTag"]
180183

181184
for token in self.normalizedTokens():
182185
new_token = token
@@ -193,13 +196,10 @@ def mainLoop(self):
193196
new_token = None
194197
else:
195198
if type in (JinjaVariableStartTag, JinjaVariableEndTag, JinjaVariable, JinjaFilter, JinjaPipe):
196-
log.debug(u"Type is a jinja variable tag")
197199
phase = self.phases["inJinjaVariable"]
198200
elif type in (JinjaStatementStartTag, JinjaStatementEndTag, JinjaStatement):
199-
log.debug(u"Type is a jinja statement tag")
200201
phase = self.phases["inJinjaStatement"]
201202
elif type in (JinjaArgumentStartTag, JinjaArgumentEndTag, JinjaArgument):
202-
log.debug(u"Type is a jinja argument tag")
203203
phase = self.phases["inJinjaArgument"]
204204
elif (
205205
len(self.tree.openElements) == 0 or
@@ -251,6 +251,12 @@ def mainLoop(self):
251251
new_token = phase.processJinjaArgumentEndTag(new_token)
252252
elif type == JinjaArgument:
253253
new_token = phase.processJinjaArgument(new_token)
254+
elif type == JinjaExtendTag:
255+
new_token = phase.processJinjaExtendTag(new_token)
256+
elif type == JinjaIncludeTag:
257+
new_token = phase.processJinjaIncludeTag(new_token)
258+
elif type == JinjaImportTag:
259+
new_token = phase.processJinjaImportTag(new_token)
254260

255261
if (type == StartTagToken and token["selfClosing"]
256262
and not token["selfClosingAcknowledged"]):
@@ -546,6 +552,18 @@ def processJinjaVariableEndTag(self, token):
546552
def processJinjaVariable(self, token):
547553
pass
548554

555+
def processJinjaExtendTag(self, token):
556+
element = self.tree.createElementWithoutNamespace(token)
557+
self.tree.openElements[-1].appendChild(element)
558+
559+
def processJinjaIncludeTag(self, token):
560+
element = self.tree.createElementWithoutNamespace(token)
561+
self.tree.openElements[-1].appendChild(element)
562+
563+
def processJinjaImportTag(self, token):
564+
element = self.tree.createElementWithoutNamespace(token)
565+
self.tree.openElements[-1].appendChild(element)
566+
549567
def processJinjaArgumentStartTag(self, token):
550568
pass
551569

‎html5lib/tests/test_jinja.py

Lines changed: 121 additions & 25 deletions
EED3
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@ def dump(tree, tabs=0):
1515

1616
class JinjaTestCase(unittest.TestCase):
1717
def setUp(self):
18-
self.parser = html5lib.HTMLParser(strict=True, namespaceHTMLElements=False)
18+
self.parser = html5lib.HTMLParser(strict=True, namespaceHTMLElements=False, tree=html5lib.treebuilders.getTreeBuilder("etree", fullTree=True))
1919

2020
def test_var_1(self):
2121
html_string = """<h1>{{ hi }}</h1>"""
@@ -197,42 +197,48 @@ def test_complete_doc(self):
197197
"""
198198

199199
tree = self.parser.parse(html_string)
200-
dump(tree)
200+
201201
self.assertTree(tree, [{
202-
'tag': 'head',
203-
'children': [{
204-
'tag': 'title',
205-
'text': 'My Webpage'
206-
}]
202+
'tag': '<!DOCTYPE>',
203+
'text': 'html'
207204
}, {
208-
'tag': 'body',
205+
'tag': 'html',
209206
'children': [{
210-
'tag': 'ul',
207+
'tag': 'head',
211208
'children': [{
212-
'tag': 'jinjafor',
213-
'value': 'item in navigation',
209+
'tag': 'title',
210+
'text': 'My Webpage'
211+
}]
212+
}, {
213+
'tag': 'body',
214+
'children': [{
215+
'tag': 'ul',
214216
'children': [{
215-
'tag': 'li',
217+
'tag': 'jinjafor',
218+
'value': 'item in navigation',
216219
'children': [{
217-
'tag': 'a',
220+
'tag': 'li',
218221
'children': [{
219-
'tag': 'jinjavariabletag',
222+
'tag': 'a',
220223
'children': [{
221-
'tag': 'jinjavariable',
222-
'value': 'item.caption'
224+
'tag': 'jinjavariabletag',
225+
'children': [{
226+
'tag': 'jinjavariable',
227+
'value': 'item.caption'
228+
}]
223229
}]
224230
}]
225231
}]
226232
}]
227-
}]
228-
}, {
229-
'tag': 'h1',
230-
'text': 'My Webpage'
231-
}, {
232-
'tag': 'jinjavariabletag',
233-
'children': [{
234-
'tag': 'jinjavariable',
235-
'value': 'a_variable'
233+
}, {
234+
'tag': 'h1',
235+
'text': 'My Webpage'
236+
}, {
237+
'tag': 'jinjavariabletag',
238+
'children': [{
239+
'tag': 'jinjavariable',
240+
'value': 'a_variable'
241+
}]
236242
}]
237243
}]
238244
}])
@@ -250,6 +256,89 @@ def test_jinja_if(self):
250256
'text': 'yay'
251257
}])
252258

259+
def test_jinja_if_lstrip(self):
260+
html_string = """
261+
{%+ if True %}yay{% endif %}
262+
"""
263+
264+
tree = self.parser.parseFragment(html_string)
265+
dump(tree)
266+
267+
self.assertTree(tree, [{
268+
'tag': 'jinjaif',
269+
'text': 'yay',
270+
'attrs': {
271+
'lstrip': False
272+
}
273+
}])
274+
275+
def test_jinja_strip_blocks(self):
276+
html_string = """
277+
{% for item in seq -%}
278+
{{ item }}
279+
{%- endfor %}
280+
"""
281+
282+
tree = self.parser.parseFragment(html_string)
283+
dump(tree)
284+
285+
self.assertTree(tree, [{
286+
'tag': 'jinjafor',
287+
'attrs': {
288+
'rstrip': True
289+
},
290+
'children': [{
291+
'tag': 'jinjavariabletag',
292+
'children': [{
293+
'tag': 'jinjavariable',
294+
'value': 'item'
295+
}]
296+
}]
297+
}])
298+
299+
def test_jinja_extend(self):
300+
html_string = """
301+
{% extends "base.html" %}
302+
"""
303+
304+
tree = self.parser.parseFragment(html_string)
305+
dump(tree)
306+
307+
self.assertTree(tree, [{
308+
'tag': 'jinjaextends',
309+
'value': '"base.html"'
310+
}])
311+
312+
def test_jinja_include(self):
313+
html_string = """
314+
{% include ['special_sidebar.html', 'sidebar.html'] ignore missing %}
< 1CF5 /td>
315+
"""
316+
317+
tree = self.parser.parseFragment(html_string)
318+
dump(tree)
319+
320+
self.assertTree(tree, [{
321+
'tag': 'jinjainclude',
322+
'value': "['special_sidebar.html', 'sidebar.html'] ignore missing"
323+
}])
324+
325+
def test_jinja_import(self):
326+
html_string = """
327+
{% import 'forms.html' as forms %}
328+
{% from 'forms.html' import input as input_field, textarea %}
329+
"""
330+
331+
tree = self.parser.parseFragment(html_string)
332+
dump(tree)
333+
334+
self.assertTree(tree, [{
335+
'tag': 'jinjaimport',
336+
'value': "'forms.html' as forms"
337+
}, {
338+
'tag': 'jinjaimport',
339+
'value': "'forms.html' import input as input_field, textarea"
340+
}])
341+
253342
def assertTree(self, root, spec):
254343
self.assertEqual(len(root), len(spec))
255344

@@ -264,3 +353,10 @@ def assertTree(self, root, spec):
264353

265354
if 'children' in spec_child:
266355
self.assertTree(child, spec_child['children'])
356+
else:
357+
self.assertEqual(len(child), 0)
358+
359+
if 'attrs' in spec_child:
360+
for k, v in spec_child['attrs'].iteritems():
361+
self.assertIn(k, child.attrib)
362+
self.assertEqual(v, child.attrib[k])

‎html5lib/tests/test_parser2.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,7 @@ def test_namespace_html_elements_1_dom(self):
4040
def test_namespace_html_elements_0_etree(self):
4141
parser = html5parser.HTMLParser(namespaceHTMLElements=True)
4242
doc = parser.parse("<html></html>")
43-
self.assertTrue(list(doc)[0].tag == "{%s}html" % (namespaces["html"],))
43+
self.assertEqual(list(doc)[0].tag, "{%s}html" % (namespaces["html"],))
4444

4545
def test_namespace_html_elements_1_etree(self):
4646
parser = html5parser.HTMLParser(namespaceHTMLElements=False)

‎html5lib/tokenizer.py

Lines changed: 42 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -328,31 +328,66 @@ def jinjaStatementStartState(self):
328328
"eof-in-jinja-statement"})
329329
self.state = self.prevState
330330
else:
331+
attrs = {}
332+
333+
if data in ['-', '+']:
334+
attrs['lstrip'] = False
335+
336+
data = self.stream.char()
337+
while data in spaceCharacters:
338+
data = self.stream.char()
339+
331340
block_type = data + self.stream.charsUntil(frozenset(("%")) | spaceCharacters)
332341

333342
block_definition = self.stream.charsUntil(frozenset(("%", "\u0000")))
334343

335344
block_definition = block_definition.strip(" \t")
336345

346+
if block_definition and block_definition[-1] == '-':
347+
attrs['rstrip'] = True
348+
block_definition = block_definition[:-1].rstrip()
349+
350+
attrs.update({
351+
"value": block_definition,
352+
"position": self.stream.position()
353+
})
354+
337355
if block_type.startswith("end"):
338356
block_type = block_type.replace("end", "")
357+
attrs['value'] = block_type.lower()
339358

340359
self.tokenQueue.append({
341360
"type": tokenTypes["JinjaStatementEndTag"],
342361
'name': u"jinja{}".format(block_type.lower()),
343-
"data": {
344-
"position": self.stream.position()
345-
},
362+
"data": attrs,
346363
"selfClosing": False
347364
})
365+
elif block_type == "extends":
366+
self.tokenQueue.append({
367+
"type": tokenTypes["JinjaExtendTag"],
368+
'name': u"jinja{}".format(block_type.lower()),
369+
"data": attrs,
370+
"selfClosing": True
371+
})
372+
elif block_type == "include":
373+
self.tokenQueue.append({
374+
"type": tokenTypes["JinjaIncludeTag"],
375+
'name': u"jinja{}".format(block_type.lower()),
376+
"data": attrs,
377+
"selfClosing": True
378+
})
379+
elif block_type in ["import", "from"]:
380+
self.tokenQueue.append({
381+
"type": tokenTypes["JinjaImportTag"],
382+
'name': u"jinjaimport",
383+
"data": attrs,
384+
"selfClosing": True
385+
})
348386
else:
349387
self.tokenQueue.append({
350388
"type": tokenTypes["JinjaStatementStartTag"],
351389
'name': u"jinja{}".format(block_type.lower()),
352-
"data": {
353-
"value": block_definition,
354-
"position": self.stream.position()
355-
},
390+
"data": attrs,
356391
"selfClosing": False
357392
})
358393

‎html5lib/treebuilders/etree.py

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -63,7 +63,6 @@ def _getAttributes(self):
6363
return self._element.attrib
6464

6565
def _setAttributes(self, attributes):
66-
log.debug(u"Attributes {}".format(attributes))
6766
# Delete existing attributes first
6867
# XXX - there may be a better way to do this...
6968
for key in list(self._element.attrib.keys()):

0 commit comments

Comments
 (0)
0