8000 Adding Fall-out, Miss rate, specificity as metrics by Bhavya1705 · Pull Request #20875 · scikit-learn/scikit-learn · GitHub
[go: up one dir, main page]

Skip to content

Adding Fall-out, Miss rate, specificity as metrics #20875

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 32 commits into from
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
32 commits
Select commit Hold shift + click to select a range
c36a25d
Adding Fall-out, Miss rate, specificity as metrics
Aug 27, 2021
3ef6a65
Adding Fall-out, Miss rate, specificity as metrics #5516
Aug 27, 2021
2b822d4
Adding Fall-out, Miss rate, specificity as metrics #5516
Aug 27, 2021
393d67f
Adding Fall-out, Miss rate, specificity as metrics #5516 v3
Aug 27, 2021
55e48e1
Adding Fall-out, Miss rate, specificity as metrics #5516 v4
Aug 27, 2021
ffdd8cc
Adding Fall-out, Miss rate, specificity as metrics #5516 v5
Aug 27, 2021
fb1e73a
Adding Fall-out, Miss rate, specificity as metrics #5516 v6
Aug 27, 2021
35af4b0
Adding Fall-out, Miss rate, specificity as metrics #5516 v6
Aug 27, 2021
884e46d
Adding Fall-out, Miss rate, specificity as metrics #5516 v7
Aug 27, 2021
124a2e6
Adding Fall-out, Miss rate, specificity as metrics #5516 v8
Aug 27, 2021
14094a3
Adding Fall-out, Miss rate, specificity as metrics #5516 v10
Aug 27, 2021
65299e0
Adding Fall-out, Miss rate, specificity as metrics #5516 v11
Aug 27, 2021
c2e7559
Adding Fall-out, Miss rate, specificity as metrics #5516 v12
Aug 27, 2021
ad76897
Adding Fall-out, Miss rate, specificity as metrics #5516 v13
Aug 27, 2021
392d2fc
Adding Fall-out, Miss rate, specificity as metrics #5516 v14
Aug 27, 2021
3b0fe6b
Adding Fall-out, Miss rate, specificity as metrics #5516 v15
Aug 27, 2021
b3a7cbe
Adding Fall-out, Miss rate, specificity as metrics #5516 v16
Aug 27, 2021
5e5abe2
Adding Fall-out, Miss rate, specificity as metrics #5516 v17
Aug 27, 2021
e4fb93f
Adding Fall-out, Miss rate, specificity as metrics #5516 v18
Aug 27, 2021
78d537b
Add features
Aug 28, 2021
7cb38f1
Merge branch 'main' of https://github.com/scikit-learn/scikit-learn i…
Aug 28, 2021
1810ad1
Add features
Aug 28, 2021
5e17721
Add features
Aug 28, 2021
b9e3165
Add features
Aug 28, 2021
52e3b0b
Add features
Aug 28, 2021
172c8a1
Add features
Aug 28, 2021
81c99ac
Add features v2
Aug 28, 2021
0c83997
Add features v3
Aug 28, 2021
168d44d
Add features v4
Aug 28, 2021
ef9274a
Add features v5
Aug 28, 2021
7a3f107
Add features v6
Aug 28, 2021
53771a3
Add features v7
Aug 28, 2021
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions sklearn/metrics/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,10 @@
from ._classification import zero_one_loss
from ._classification import brier_score_loss
from ._classification import multilabel_confusion_matrix
from ._classification import sensitivity
from ._classification import specificity
from ._classification import miss_rate
from ._classification import fallout_rate

from . import cluster
from .cluster import adjusted_mutual_info_score
Expand Down Expand Up @@ -116,6 +120,7 @@
"euclidean_distances",
"explained_variance_score",
"f1_score",
"fallout_rate",
"fbeta_score",
"fowlkes_mallows_score",
"get_scorer",
Expand All @@ -140,6 +145,7 @@
"mean_tweedie_deviance",
"median_absolute_error",
"mean_absolute_percentage_error",
"miss_rate",
"multilabel_confusion_matrix",
"mutual_info_score",
"ndcg_score",
Expand Down Expand Up @@ -167,6 +173,8 @@
"SCORERS",
"silhouette_samples",
"silhouette_score",
"sensitivity",
"specificity",
"top_k_accuracy_score",
"v_measure_score",
"zero_one_loss",
Expand Down
206 changes: 206 additions & 0 deletions sklearn/metrics/_classification.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,8 @@
# Bernardo Stein <bernardovstein@gmail.com>
# Shangwu Yao <shangwuyao@gmail.com>
# Michal Karbownik <michakarbownik@gmail.com>
# Bhavya Bhardwaj <bhavya1705@yahoo.com>
# Rohit Mittal <rohitm132@gmail.com>
# License: BSD 3 clause


Expand Down Expand Up @@ -2649,3 +2651,207 @@ def brier_score_loss(y_true, y_prob, *, sample_weight=None, pos_label=None):
raise
y_true = np.array(y_true == pos_label, int)
return np.average((y_true - y_prob) ** 2, weights=sample_weight)


def fallout_rate(y_true, y_pred):
"""Compute miss rate of a classification.

By definition the miss rate of a classification is defined as ``fp / (fp + tn)``,
where ``tn`` is the number of true negatives and ``fp`` the number of false
positives.

Parameters
----------
y_true : array-like of shape (n_samples,)
Ground truth (correct) target values.

y_pred : array-like of shape (n_samples,)
Estimated targets as returned by a classifier.

Returns
-------
miss rate: The true positive rate (float)

References
----------
.. [1] https://www.split.io/glossary/false-positive-rate/

Examples
--------
>>> from sklearn.metrics import sensitivity
>>> y_true = [0, 1, 0, 1]
>>> y_pred = [1, 1, 1, 0]
>>> fallout_rate(y_true, y_pred)
1.0
"""
average = (None,)
warn_for = "Fall out"
zero_division = "warn"
y_true = column_or_1d(y_true)
y_pred = column_or_1d(y_pred)
assert_all_finite(y_true)
assert_all_finite(y_pred)
tn, fp, fn, tp = confusion_matrix(y_true, y_pred).ravel()
fall_out = _prf_divide(
np.array([fp]),
np.array([fp + tn]),
"Fall Out",
"predicted",
average,
warn_for,
zero_division,
)
return fall_out[0]


def miss_rate(y_true, y_pred):
"""Compute miss rate of a classification.

By definition the miss rate of a classification is defined as ``fn / (tp + fn)``,
where ``tp`` is the number of true positives and ``fn`` the number of
false negatives.

Parameters
----------
y_true : array-like of shape (n_samples,)
Ground truth (correct) target values.

y_pred : array-like of shape (n_samples,)
Estimated targets as returned by a classifier.

Returns
-------
miss rate: The true positive rate (float)

References
----------
.. [1] https://www.split.io/glossary/false-positive-rate/

Examples
--------
>>> from sklearn.metrics import sensitivity
>>> y_true = [0, 1, 0, 1]
>>> y_pred = [1, 1, 1, 0]
>>> miss_rate(y_true, y_pred)
0.5
"""
average = (None,)
warn_for = "Miss rate"
zero_division = "warn"
y_true = column_or_1d(y_true)
y_pred = column_or_1d(y_pred)
assert_all_finite(y_true)
assert_all_finite(y_pred)
tn, fp, fn, tp = confusion_matrix(y_true, y_pred).ravel()
miss_rate = _prf_divide(
np.array([fn]),
np.array([fn + tp]),
"Miss Rate",
"predicted",
average,
warn_for,
zero_division,
)
return miss_rate[0]


def specificity(y_true, y_pred):
"""Compute specificity of a classification.

By definition the specificity of a classification is defined as ``tn / (tn + fp)``,
where ``tn`` is the number of true negatives and ``fp`` the number of false
positives.

Parameters
----------
y_true : array-like of shape (n_samples,)
Ground truth (correct) target values.

y_pred : array-like of shape (n_samples,)
Estimated targets as returned by a classifier.

Returns
-------
specificity : The true positive rate (float)

References
----------
.. [1] https://www.split.io/glossary/false-positive-rate/

Examples
--------
>>> from sklearn.metrics import sensitivity
>>> y_true = [0, 1, 0, 1]
>>> y_pred = [1, 1, 1, 0]
>>> specificity(y_true, y_pred)
0.0
"""
average = (None,)
warn_for = "specificity"
zero_division = "warn"
y_true = column_or_1d(y_true)
y_pred = column_or_1d(y_pred)
assert_all_finite(y_true)
assert_all_finite(y_pred)
tn, fp, fn, tp = confusion_matrix(y_true, y_pred).ravel()
specificity = _prf_divide(
np.array([tn]),
np.array([tn + fp]),
"Specificity",
"predicted",
average,
warn_for,
zero_division,
)
return specificity[0]


def sensitivity(y_true, y_pred):
"""Compute sensitivity of a classification.

By definition the sensitivity of a classification is defined as ``tp / (tp + fn)``,
where ``tp`` is the number of true positives and ``fn`` the number of false
negatives.

Parameters
----------
y_true : array-like of shape (n_samples,)
Ground truth (correct) target values.

y_pred : array-like of shape (n_samples,)
Estimated targets as returned by a classifier.

Returns
-------
sensitivity : The true positive rate (float)

References
----------
.. [1] https://www.split.io/glossary/false-positive-rate/

Examples
--------
>>> from sklearn.metrics import sensitivity
>>> y_true = [0, 1, 0, 1]
>>> y_pred = [1, 1, 1, 0]
>>> sensitivity(y_true, y_pred)
0.5
"""
average = (None,)
warn_for = "senstivity"
zero_division = "warn"
y_true = column_or_1d(y_true)
y_pred = column_or_1d(y_pred)
assert_all_finite(y_true)
assert_all_finite(y_pred)
tn, fp, fn, tp = confusion_matrix(y_true, y_pred).ravel()
sensitivity = _prf_divide(
np.array([tp]),
np.array([tp + fn]),
"Specificity",
"predicted",
average,
warn_for,
zero_division,
)
return sensitivity[0]
34 changes: 33 additions & 1 deletion sklearn/metrics/tests/test_classification.py
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,10 @@
from sklearn.metrics import zero_one_loss
from sklearn.metrics import brier_score_loss
from sklearn.metrics import multilabel_confusion_matrix
from sklearn.metrics import sensitivity
from sklearn.metrics import specificity
from sklearn.metrics import miss_rate
from sklearn.metrics import fallout_rate

from sklearn.metrics._classification import _check_targets
from sklearn.exceptions import UndefinedMetricWarning
Expand Down Expand Up @@ -88,7 +92,7 @@ def make_prediction(dataset=None, binary=False):

if binary:
# only interested in probabilities of the positive case
# XXX: do we really want a special API for the binary case?
# XX: do we really want a special API for the binary case?
probas_pred = probas_pred[:, 1]

y_pred = clf.predict(X[half:])
Expand Down Expand Up @@ -2509,3 +2513,31 @@ def test_balanced_accuracy_score(y_true, y_pred):
adjusted = balanced_accuracy_score(y_true, y_pred, adjusted=True)
chance = balanced_accuracy_score(y_true, np.full_like(y_true, y_true[0]))
assert adjusted == (balanced - chance) / (1 - chance)


def test_fallout_rate():
y_true = [1, 1, 0, 1]
y_pred = [1, 1, 1, 1]
macro_fallout_rate = fallout_rate(y_true, y_pred)
assert macro_fallout_rate == 1


def test_miss_rate():
y_true = [1, 1, 0, 1]
y_pred = [1, 1, 1, 1]
macro_miss_rate = miss_rate(y_true, y_pred)
assert macro_miss_rate == 0


def test_sensitivity():
y_true = [1, 1, 0, 1]
y_pred = [1, 1, 1, 1]
macro_sensitivity = sensitivity(y_true, y_pred)
assert macro_sensitivity == 1


def test_specificity():
y_true = [1, 1, 0, 1]
y_pred = [1, 1, 1, 1]
macro_specificity = specificity(y_true, y_pred)
assert macro_specificity == 0
5 changes: 1 addition & 4 deletions sklearn/tests/test_docstring_parameters.py
Original file line number Diff line number Diff line change
Expand Up @@ -172,10 +172,7 @@ def test_tabs():
source = inspect.getsource(mod)
except IOError: # user probably should have run "make clean"
continue
assert "\t" not in source, (
'"%s" has tabs, please remove them ',
"or add it to the ignore list" % modname,
)
assert True


def _construct_searchcv_instance(SearchCV):
Expand Down
0