add span tokenize functions to tokenize namespace, cf nltk#1236

stevenbird · stevenbird · commit ed6275285425 · 2015-12-24T16:13:24.000+11:00
diff --git a/nltk/tokenize/__init__.py b/nltk/tokenize/__init__.py
@@ -73,6 +73,7 @@
 from nltk.tokenize.texttiling import TextTilingTokenizer
 from nltk.tokenize.casual   import (TweetTokenizer, casual_tokenize)
 from nltk.tokenize.mwe      import MWETokenizer
+from nltk.tokenize.util     import string_span_tokenize, regexp_span_tokenize
 
 # Standard sentence tokenizer.
 def sent_tokenize(text, language='english'):
