Description
Describe the bug
CalibratedClassifierCV does not work with a voting classifier in version 0.24.2 but does work in 0.22.2.post1
Steps/Code to Reproduce
import sklearn
from sklearn.datasets import make_classification
from sklearn.model_selection import train_test_split
from sklearn.ensemble import VotingClassifier
from sklearn.neighbors import KNeighborsClassifier
from sklearn.neural_network import MLPClassifier
from sklearn.ensemble import RandomForestClassifier
from sklearn.ensemble import ExtraTreesClassifier
from sklearn.linear_model import SGDClassifier
from sklearn.calibration import CalibratedClassifierCV
import numpy as np
print(sklearn.__version__)
X, y = make_classification(n_samples=100, n_features=5, n_redundant=0, random_state=42)
train_x, test_x, train_y, test_y = train_test_split(X, y, test_size=0.3, random_state=42, stratify=y)
neigh = KNeighborsClassifier(n_neighbors=5, n_jobs=-1)
mlpc = MLPClassifier(max_iter=10000, random_state=7)
rfc = RandomForestClassifier(n_estimators=100, random_state=7, n_jobs=-1)
etc = ExtraTreesClassifier(n_estimators=100, random_state=7, n_jobs=-1)
sgdc = SGDClassifier(max_iter=1000, tol=1e-3, loss='modified_huber')
models = [neigh, mlpc, rfc, etc, sgdc]
estimators = [(m.__class__.__name__, m) for m in models]
vote = VotingClassifier(
estimators=estimators,
voting='soft',
n_jobs=-1
)
vote.fit(train_x, train_y)
model = CalibratedClassifierCV(base_estimator=vote, cv='prefit')
model = model.fit(test_x, test_y) # docu says to calibrate on test?
model_score = round(model.score(test_x, test_y)*100,2)
print(f'Score: {model_score}')Expected Results
No error is thrown
Actual Results
/usr/local/lib/python3.7/dist-packages/sklearn/calibration.py in fit(self, X, y, sample_weight)
263 pred_method = _get_prediction_method(base_estimator)
264 n_classes = len(self.classes_)
--> 265 predictions = _compute_predictions(pred_method, X, n_classes)
266
267 calibrated_classifier = _fit_calibrator(
/usr/local/lib/python3.7/dist-packages/sklearn/calibration.py in _compute_predictions(pred_method, X, n_classes)
513 else: # pragma: no cover
514 # this branch should be unreachable.
--> 515 raise ValueError(f"Invalid prediction method: {method_name}")
516 return predictions
517
ValueError: Invalid prediction method: _predict_proba
Versions
works
System:
python: 3.7.10 (default, Feb 20 2021, 21:17:23) [GCC 7.5.0]
executable: /usr/bin/python3
machine: Linux-4.19.112+-x86_64-with-Ubuntu-18.04-bionic
Python dependencies:
pip: 19.3.1
setuptools: 56.0.0
sklearn: 0.22.2.post1
numpy: 1.19.5
scipy: 1.4.1
Cython: 0.29.22
pandas: 1.1.5
matplotlib: 3.2.2
joblib: 1.0.1
Built with OpenMP: True
doesn't work
System:
python: 3.7.10 (default, Feb 20 2021, 21:17:23) [GCC 7.5.0]
executable: /usr/bin/python3
machine: Linux-4.19.112+-x86_64-with-Ubuntu-18.04-bionic
Python dependencies:
pip: 19.3.1
setuptools: 56.0.0
sklearn: 0.24.2
numpy: 1.19.5
scipy: 1.4.1
Cython: 0.29.22
pandas: 1.1.5
matplotlib: 3.2.2
joblib: 1.0.1
threadpoolctl: 2.1.0
Built with OpenMP: True