Fix changing encoding to actually change encoding; add test for it

gsnedders · gsnedders · commit 1d9f391f6f92 · 2016-05-05T00:12:53.000+01:00
diff --git a/html5lib/inputstream.py b/html5lib/inputstream.py
@@ -509,8 +509,8 @@ def changeEncoding(self, newEncoding):
             self.charEncoding = (self.charEncoding[0], "certain")
         else:
             self.rawStream.seek(0)
-            self.reset()
             self.charEncoding = (newEncoding, "certain")
+            self.reset()
             raise ReparseException("Encoding changed from %s to %s" % (self.charEncoding[0], newEncoding))
 
     def detectBOM(self):
diff --git a/html5lib/tests/test_encoding.py b/html5lib/tests/test_encoding.py
@@ -21,6 +21,19 @@ def test_basic_prescan_length():
     assert 'utf-8' == stream.charEncoding[0].name
 
 
+def test_parser_reparse():
+    data = "<title>Caf\u00E9</title><!--a--><meta charset='utf-8'>".encode('utf-8')
+    pad = 10240 - len(data) + 1
+    data = data.replace(b"-a-", b"-" + (b"a" * pad) + b"-")
+    assert len(data) == 10240  # Sanity
+    stream = inputstream.HTMLBinaryInputStream(data, chardet=False)
+    assert 'windows-1252' == stream.charEncoding[0].name
+    p = HTMLParser(namespaceHTMLElements=False)
+    doc = p.parse(data, useChardet=False)
+    assert 'utf-8' == p.documentEncoding
+    assert doc.find(".//title").text == "Caf\u00E9"
+
+
 def runParserEncodingTest(data, encoding):
     p = HTMLParser()
     assert p.documentEncoding is None
