Avoid reading the whole input when using chardet (by reading chunks of bytes and feeding them to the chardet UniversalDetector until it detects the encoding with enough confidence)

tbroyer · tbroyer · commit 5dea6014e21d · 2007-06-23T15:16:55.000Z
--HG--
extra : convert_revision : svn%3Aacbfec75-9323-0410-a652-858a13e371e0/trunk%40809
diff --git a/src/inputstream.py b/src/inputstream.py
@@ -40,6 +40,8 @@ def __init__(self, source, encoding=None, parseMeta=True, chardet=True):
         #Number of bytes to use when looking for a meta element with
         #encoding information
         self.numBytesMeta = 512
+        #Number of bytes to use when using detecting encoding using chardet
+        self.numBytesChardet = 100
         #Encoding to use if no other information can be found
         self.defaultEncoding = "windows-1252"
         
@@ -84,10 +86,18 @@ def detectEncoding(self, parseMeta=True, chardet=True):
         #Guess with chardet, if avaliable
         if encoding is None and chardet:
             try:
-                import chardet
-                buffer = self.rawStream.read()
-                encoding = chardet.detect(buffer)['encoding']
-                self.seek(buffer, 0)
+                from chardet.universaldetector import UniversalDetector
+                buffers = []
+                detector = UniversalDetector()
+                while not detector.done:
+                    buffer = self.rawStream.read(self.numBytesChardet)
+                    if not buffer:
+                        break
+                    buffers.append(buffer)
+                    detector.feed(buffer)
+                detector.close()
+                encoding = detector.result['encoding']
+                self.seek("".join(buffers), 0)
             except ImportError:
                 pass
         # If all else fails use the default encoding
