seckcoder
diff --git a/‎sklearn/feature_extraction/tests/test_text.py
Lines changed: 22 additions & 0 deletions b/‎sklearn/feature_extraction/tests/test_text.py
Lines changed: 22 additions & 0 deletions
diff --git a/‎sklearn/feature_extraction/text.py
Lines changed: 2 additions & 2 deletions b/‎sklearn/feature_extraction/text.py
Lines changed: 2 additions & 2 deletions
@@ -580,6 +580,28 @@ def test_vectorizer_pipeline_grid_selection():
     assert_false(best_vectorizer.fixed_vocabulary)
 
 
+def test_count_vectorizer_unicode():
+    # tests that the count vectorizer works with cyrillic.
+    document = (u"\xd0\x9c\xd0\xb0\xd1\x88\xd0\xb8\xd0\xbd\xd0\xbd\xd0\xbe\xd0"
+        u"\xb5 \xd0\xbe\xd0\xb1\xd1\x83\xd1\x87\xd0\xb5\xd0\xbd\xd0\xb8\xd0"
+        u"\xb5 \xe2\x80\x94 \xd0\xbe\xd0\xb1\xd1\x88\xd0\xb8\xd1\x80\xd0\xbd"
+        u"\xd1\x8b\xd0\xb9 \xd0\xbf\xd0\xbe\xd0\xb4\xd1\x80\xd0\xb0\xd0\xb7"
+        u"\xd0\xb4\xd0\xb5\xd0\xbb \xd0\xb8\xd1\x81\xd0\xba\xd1\x83\xd1\x81"
+        u"\xd1\x81\xd1\x82\xd0\xb2\xd0\xb5\xd0\xbd\xd0\xbd\xd0\xbe\xd0\xb3"
+        u"\xd0\xbe \xd0\xb8\xd0\xbd\xd1\x82\xd0\xb5\xd0\xbb\xd0\xbb\xd0"
+        u"\xb5\xd0\xba\xd1\x82\xd0\xb0, \xd0\xb8\xd0\xb7\xd1\x83\xd1\x87"
+        u"\xd0\xb0\xd1\x8e\xd1\x89\xd0\xb8\xd0\xb9 \xd0\xbc\xd0\xb5\xd1\x82"
+        u"\xd0\xbe\xd0\xb4\xd1\x8b \xd0\xbf\xd0\xbe\xd1\x81\xd1\x82\xd1\x80"
+        u"\xd0\xbe\xd0\xb5\xd0\xbd\xd0\xb8\xd1\x8f \xd0\xb0\xd0\xbb\xd0\xb3"
+        u"\xd0\xbe\xd1\x80\xd0\xb8\xd1\x82\xd0\xbc\xd0\xbe\xd0\xb2, \xd1\x81"
+        u"\xd0\xbf\xd0\xbe\xd1\x81\xd0\xbe\xd0\xb1\xd0\xbd\xd1\x8b\xd1\x85 "
+        u"\xd0\xbe\xd0\xb1\xd1\x83\xd1\x87\xd0\xb0\xd1\x82\xd1\x8c\xd1\x81\xd1"
+        u"\x8f.")
+    vect = CountVectorizer(min_df=1)
+    X = vect.fit_transform([document])
+    assert_equal(X.shape, (1, 15))
+
+
 def test_tfidf_vectorizer_with_fixed_vocabulary():
     # non regression smoke test for inheritance issues
     vocabulary = ['pizza', 'celeri']
 
@@ -207,7 +207,7 @@ class CountVectorizer(BaseEstimator):
     def __init__(self, input='content', charset='utf-8',
                  charset_error='strict', strip_accents=None,
                  lowercase=True, preprocessor=None, tokenizer=None,
-                 stop_words=None, token_pattern=ur"\b\w\w+\b",
+                 stop_words=None, token_pattern=ur"(?u)\b\w\w+\b",
                  ngram_range=(1, 1),
                  min_n=None, max_n=None, analyzer='word',
                  max_df=1.0, min_df=2, max_features=None,
@@ -830,7 +830,7 @@ class TfidfVectorizer(CountVectorizer):
     def __init__(self, input='content', charset='utf-8',
             charset_error='strict', strip_accents=None, lowercase=True,
             preprocessor=None, tokenizer=None, analyzer='word',
-            stop_words=None, token_pattern=ur"\b\w\w+\b", min_n=None,
+            stop_words=None, token_pattern=ur"(?u)\b\w\w+\b", min_n=None,
             max_n=None, ngram_range=(1, 1), max_df=1.0, min_df=2,
             max_features=None, vocabulary=None, binary=False, dtype=long,
             norm='l2', use_idf=True, smooth_idf=True, sublinear_tf=False):