Add handling of 0-length regex for regexp_span_tokenize for consistency with string_span_tokenize

erip · erip · commit d39337b94ccc · 2015-12-19T16:34:55.000-05:00
diff --git a/nltk/tokenize/util.py b/nltk/tokenize/util.py
@@ -58,6 +58,8 @@ def regexp_span_tokenize(s, regexp):
     :type regexp: str
     :rtype: iter(tuple(int, int))
     """
+    if len(regexp) == 0:
+        raise ValueError("regexp must not be empty")
     left = 0
     for m in finditer(regexp, s):
         right, next = m.span()
