8000 MNT Cleaner cython cdef loss function in SGD by lorentzenchr · Pull Request #17191 · scikit-learn/scikit-learn · GitHub
[go: up one dir, main page]
Skip to content

MNT Cleaner cython cdef loss function in SGD #17191

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
May 12, 2020
Merged

MNT Cleaner cython cdef loss function in SGD #17191

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
e612e68
MNT cleaner cdef loss function in sgd
May 12, 2020
6e99dbd
MNT clearify comments for cdef and nogil
May 12, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 4 additions & 4 deletions sklearn/linear_model/_sag_fast.pyx.tp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -152,7 +152,7 @@ cdef class MultinomialLogLoss{{name}}:
loss = (logsumexp_prediction - prediction[int(y)]) * sample_weight
return loss

cdef void _dloss(self, {{c_type}}* prediction, {{c_type}} y, int n_classes,
cdef void dloss(self, {{c_type}}* prediction, {{c_type}} y, int n_classes,
{{c_type}} sample_weight, {{c_type}}* gradient_ptr) nogil:
r"""Multinomial Logistic regression gradient of the loss.

Expand Down Expand Up @@ -419,10 +419,10 @@ def sag{{name}}(SequentialDataset{{name}} dataset,

# compute the gradient for this sample, given the prediction
if multinomial:
multiloss._dloss(prediction, y, n_classes, sample_weight,
multiloss.dloss(prediction, y, n_classes, sample_weight,
gradient)
else:
gradient[0] = loss._dloss(prediction[0], y) * sample_weight
gradient[0] = loss.dloss(prediction[0], y) * sample_weight

# L2 regularization by simply rescaling the weights
wscale *= wscale_update
Expand Down Expand Up @@ -783,7 +783,7 @@ def _multinomial_grad_loss_all_samples(
intercept, prediction, n_classes)

# compute the gradient for this sample, given the prediction
multiloss._dloss(prediction, y, n_classes, sample_weight, gradient)
multiloss.dloss(prediction, y, n_classes, sample_weight, gradient)

# compute the loss for this sample, given the prediction
sum_loss += multiloss._loss(prediction, y, n_classes, sample_weight)
Expand Down
10 changes: 5 additions & 5 deletions sklearn/linear_model/_sgd_fast.pxd
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,24 +3,24 @@

cdef class LossFunction:
cdef double loss(self, double p, double y) nogil
cdef double _dloss(self, double p, double y) nogil
cdef double dloss(self, double p, double y) nogil


cdef class Regression(LossFunction):
cdef double loss(self, double p, double y) nogil
cdef double _dloss(self, double p, double y) nogil
cdef double dloss(self, double p, double y) nogil


cdef class Classification(LossFunction):
cdef double loss(self, double p, double y) nogil
cdef double _dloss(self, double p, double y) nogil
cdef double dloss(self, double p, double y) nogil


cdef class Log(Classification):
cdef double loss(self, double p, double y) nogil
cdef double _dloss(self, double p, double y) nogil
cdef double dloss(self, double p, double y) nogil


cdef class SquaredLoss(Regression):
cdef double loss(self, double p, double y) nogil
cdef double _dloss(self, double p, double y) nogil
cdef double dloss(self, double p, double y) nogil
36 changes: 19 additions & 17 deletions sklearn/linear_model/_sgd_fast.pyx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,14 @@ cdef class LossFunction:
"""
return 0.

def dloss(self, double p, double y):
def py_dloss(self, double p, double y):
"""Python version of `dloss` for testing.

Pytest needs a python function and can't use cdef functions.
"""
return self.dloss(p, y)

cdef double dloss(self, double p, double y) nogil:
"""Evaluate the derivative of the loss function with respect to
the prediction `p`.

Expand All @@ -81,11 +88,6 @@ cdef class LossFunction:
double
The derivative of the loss function with regards to `p`.
"""
return self._dloss(p, y)

cdef double _dloss(self, double p, double y) nogil:
# Implementation of dloss; separate function because cpdef and nogil
# can't be combined.
return 0.


Expand All @@ -95,7 +97,7 @@ cdef class Regression(LossFunction):
cdef double loss(self, double p, double y) nogil:
return 0.

cdef double _dloss(self, double p, double y) nogil:
cdef double dloss(self, double p, double y) nogil:
return 0.


Expand All @@ -105,7 +107,7 @@ cdef class Classification(LossFunction):
cdef double loss(self, double p, double y) nogil:
return 0.

cdef double _dloss(self, double p, double y) nogil:
cdef double dloss(self, double p, double y) nogil:
return 0.


Expand All @@ -126,7 +128,7 @@ cdef class ModifiedHuber(Classification):
else:
return -4.0 * z

cdef double _dloss(self, double p, double y) nogil:
cdef double dloss(self, double p, double y) nogil:
cdef double z = p * y
if z >= 1.0:
return 0.0
Expand Down Expand Up @@ -161,7 +163,7 @@ cdef class Hinge(Classification):
return self.threshold - z
return 0.0

cdef double _dloss(self, double p, double y) nogil:
cdef double dloss(self, double p, double y) nogil:
cdef double z = p * y
if z <= self.threshold:
return -y
Expand Down Expand Up @@ -193,7 +195,7 @@ cdef class SquaredHinge(Classification):
return z * z
return 0.0

cdef double _dloss(self, double p, double y) nogil:
cdef double dloss(self, double p, double y) nogil:
cdef double z = self.threshold - p * y
if z > 0:
return -2 * y * z
Expand All @@ -215,7 +217,7 @@ cdef class Log(Classification):
return -z
return log(1.0 + exp(-z))

cdef double _dloss(self, double p, double y) nogil:
cdef double dloss(self, double p, double y) nogil:
cdef double z = p * y
# approximately equal and saves the computation of the log
if z > 18.0:
Expand All @@ -233,7 +235,7 @@ cdef class SquaredLoss(Regression):
cdef double loss(self, double p, double y) nogil:
return 0.5 * (p - y) * (p - y)

cdef double _dloss(self, double p, double y) nogil:
cdef double dloss(self, double p, double y) nogil:
return p - y

def __reduce__(self):
Expand Down Expand Up @@ -262,7 +264,7 @@ cdef class Huber(Regression):
else:
return self.c * abs_r - (0.5 * self.c * self.c)

cdef double _dloss(self, double p, double y) nogil:
cdef double dloss(self, double p, double y) nogil:
cdef double r = p - y
cdef double abs_r = fabs(r)
if abs_r <= self.c:
Expand Down Expand Up @@ -291,7 +293,7 @@ cdef class EpsilonInsensitive(Regression):
cdef double ret = fabs(y - p) - self.epsilon
return ret if ret > 0 else 0

cdef double _dloss(self, double p, double y) nogil:
cdef double dloss(self, double p, double y) nogil:
if y - p > self.epsilon:
return -1
elif p - y > self.epsilon:
Expand All @@ -318,7 +320,7 @@ cdef class SquaredEpsilonInsensitive(Regression):
cdef double ret = fabs(y - p) - self.epsilon
return ret * ret if ret > 0 else 0

cdef double _dloss(self, double p, double y) nogil:
cdef double dloss(self, double p, double y) nogil:
cdef double z
z = y - p
if z > self.epsilon:
Expand Down Expand Up @@ -542,7 +544,7 @@ def _plain_sgd(np.ndarray[double, ndim=1, mode='c'] weights,
update = sqnorm(x_data_ptr, x_ind_ptr, xnnz)
update = loss.loss(p, y) / (update + 0.5 / C)
else:
dloss = loss._dloss(p, y)
dloss = loss.dloss(p, y)
# clip dloss with large values to avoid numerical
# instabilities
if dloss < -MAX_DLOSS:
Expand Down
6 changes: 3 additions & 3 deletions sklearn/linear_model/tests/test_sgd.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1438,7 +1438,7 @@ def _test_gradient_common(loss_function, cases):
# Test gradient of different loss functions
# cases is a list of (p, y, expected)
for p, y, expected in cases:
assert_almost_equal(loss_function.dloss(p, y), expected)
assert_almost_equal(loss_function.py_dloss(p, y), expected)


def test_gradient_hinge():
Expand Down Expand Up @@ -1488,8 +1488,8 @@ def test_gradient_log():
(17.9, -1.0, 1.0), (-17.9, 1.0, -1.0),
]
_test_gradient_common(loss, cases)
assert_almost_equal(loss.dloss(18.1, 1.0), np.exp(-18.1) * -1.0, 16)
assert_almost_equal(loss.dloss(-18.1, -1.0), np.exp(-18.1) * 1.0, 16)
assert_almost_equal(loss.py_dloss(18.1, 1.0), np.exp(-18.1) * -1.0, 16)
assert_almost_equal(loss.py_dloss(-18.1, -1.0), np.exp(-18.1) * 1.0, 16)


def test_gradient_squared_loss():
Expand Down
0