Generate parse errors for nulls encountered in the input stream

rubys · rubys · commit 2350d5694dae · 2007-06-27T11:34:49.000Z
--HG--
extra : convert_revision : svn%3Aacbfec75-9323-0410-a652-858a13e371e0/trunk%40833
diff --git a/src/html5lib/inputstream.py b/src/html5lib/inputstream.py
@@ -53,6 +53,7 @@ def __init__(self, source, encoding=None, parseMeta=True, chardet=True):
         self.dataStream = codecs.getreader(self.charEncoding)(self.rawStream, 'replace')
 
         self.queue = []
+        self.errors = []
 
         self.line = self.col = 0
         self.lineLengths = []
@@ -214,7 +215,10 @@ def char(self):
                 return EOF
 
             # Normalize newlines and null characters
-            if c == '\x00': c = u'\uFFFD'
+            if c == '\x00':
+                self.errors.append('null character found in input stream, '
+                  'replaced with U+FFFD')
+                c = u'\uFFFD'
             if c == '\r':
                 c = self.dataStream.read(1, 1)
                 if c != '\n':
diff --git a/src/html5lib/tokenizer.py b/src/html5lib/tokenizer.py
@@ -93,6 +93,8 @@ def __iter__(self):
         # Start processing. When EOF is reached self.state will return False
         # instead of True and the loop will terminate.
         while self.state():
+            while self.stream.errors:
+                yield {"type": "ParseError", "data": self.stream.errors.pop(0)}
             while self.tokenQueue:
                 yield self.tokenQueue.pop(0)
 
