BUG: AUC should not assume curve is increasing

GaelVaroquaux · GaelVaroquaux · commit 9f18586ebfc8 · 2012-11-08T17:24:14.000+01:00
While an ROC curve is increasing, a precision-recall is not
diff --git a/sklearn/metrics/metrics.py b/sklearn/metrics/metrics.py
@@ -213,14 +213,14 @@ def average_precision_score(y_true, y_score):
     auc_score: Area under the ROC curve
     """
     precision, recall, thresholds = precision_recall_curve(y_true, y_score)
-
     return auc(recall, precision)
 
 
 def auc_score(y_true, y_score):
     """Compute Area Under the Curve (AUC) from prediction scores.
 
-    Note: this implementation is restricted to the binary classification task.
+    Note: this implementation is restricted to the binary classification
+    task.
 
     Parameters
     ----------
@@ -246,10 +246,10 @@ def auc_score(y_true, y_score):
     """
 
     fpr, tpr, tresholds = roc_curve(y_true, y_score)
-    return auc(fpr, tpr)
+    return auc(fpr, tpr, reorder=True)
 
 
-def auc(x, y):
+def auc(x, y, reorder=False):
     """Compute Area Under the Curve (AUC) using the trapezoidal rule
 
     This is a general fuction, given points on a curve.
@@ -263,6 +263,11 @@ def auc(x, y):
     y : array, shape = [n]
         y coordinates
 
+    reorder : boolean, optional
+        If True, assume that the curve is ascending in the case of ties,
+        as for an ROC curve. With descending curve, you will get false
+        results
+
     Returns
     -------
     auc : float
@@ -287,10 +292,18 @@ def auc(x, y):
         raise ValueError('At least 2 points are needed to compute'
                          ' area under curve, but x.shape = %s' % x.shape)
 
-    # reorder the data points according to the x axis and using y to break ties
-    x, y = np.array(sorted(points for points in zip(x, y))).T
+    if reorder:
+        # reorder the data points according to the x axis and using y to
+        # break ties
+        x, y = np.array(sorted(points for points in zip(x, y))).T
+        h = np.diff(x)
+    else:
+        h = np.diff(x)
+        if np.any(h < 0):
+            h *= -1
+            assert not np.any(h < 0), ("Reordering is not turned on, and "
+                        "The x array is not increasing: %s" % x)
 
-    h = np.diff(x)
     area = np.sum(h * (y[1:] + y[:-1])) / 2.0
     return area
 
diff --git a/sklearn/metrics/tests/test_metrics.py b/sklearn/metrics/tests/test_metrics.py
@@ -156,13 +156,12 @@ def test_auc():
 
 
 def test_auc_duplicate_values():
-    """Test Area Under Curve (AUC) computation with duplicate values
+    # Test Area Under Curve (AUC) computation with duplicate values
 
-    auc() was previously sorting the x and y arrays according to the indices
-    from numpy.argsort(x), which was reordering the tied 0's in this example
-    and resulting in an incorrect area computation. This test detects the
-    error.
-    """
+    # auc() was previously sorting the x and y arrays according to the indices
+    # from numpy.argsort(x), which was reordering the tied 0's in this example
+    # and resulting in an incorrect area computation. This test detects the
+    # error.
     x = [0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.5, 1.]
     y = [0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9,
          1., 1., 1., 1., 1., 1., 1., 1.]
@@ -201,6 +200,17 @@ def test_precision_recall_f1_score_binary():
     assert_array_almost_equal(fs, 0.74, 2)
 
 
+def test_average_precision_score_duplicate_values():
+    # Duplicate values with precision-recall require a different
+    # processing than when computing the AUC of a ROC, because the
+    # precision-recall curve is a decreasing curve
+    # The following situtation corresponds to a perfect
+    # test statistic, the average_precision_score should be 1
+    y_true  = [ 0,  0,  0,  0,  1,  1,  1,  1,  1,  1,  1]
+    y_score = [ 0, .1, .1, .5, .5, .6, .6, .9, .9,  1,  1]
+    assert_equal(average_precision_score(y_true, y_score), 1)
+
+
 def test_precision_recall_fscore_support_errors():
     y_true, y_pred, _ = make_prediction(binary=True)
 
