8000 ENH Support float32 in SGDClassifier and SGDRegressor (#25587) · scikit-learn/scikit-learn@1a669d8 · GitHub
[go: up one dir, main page]
Skip to content

Commit 1a669d8

Browse files
OmarManzoorOmarManzoor
authored
ENH Support float32 in SGDClassifier and SGDRegressor (#25587)
1 parent 90d7e41 commit 1a669d8

File tree

6 files changed

+235
-82
lines changed

6 files changed

+235
-82
lines changed

‎.gitignore

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -85,6 +85,7 @@ sklearn/utils/_seq_dataset.pxd
8585
sklearn/utils/_weight_vector.pyx
8686
sklearn/utils/_weight_vector.pxd
8787
sklearn/linear_model/_sag_fast.pyx
88+
sklearn/linear_model/_sgd_fast.pyx
8889
sklearn/metrics/_dist_metrics.pyx
8990
sklearn/metrics/_dist_metrics.pxd
9091
sklearn/metrics/_pairwise_distances_reduction/_argkmin.pxd

‎doc/whats_new/v1.3.rst

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -159,6 +159,13 @@ Changelog
159159
- |Enhancement| Added the parameter `fill_value` to :class:`impute.IterativeImputer`.
160160
:pr:`25232` by :user:`Thijs van Weezel <ValueInvestorThijs>`.
161161

162+
:mod:`sklearn.linear_model`
163+
...........................
164+
165+
- |Enhancement| :class:`SGDClassifier`, :class:`SGDRegressor` and
166+
:class:`SGDOneClassSVM` now preserve dtype for `numpy.float32`.
167+
:pr:`25587` by :user:`Omar Salman <OmarManzoor>`
168+
162169
:mod:`sklearn.metrics`
163170
......................
164171

‎setup.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -309,7 +309,7 @@ def check_package_status(package, min_version):
309309
],
310310
"linear_model": [
311311
{"sources": ["_cd_fast.pyx"], "include_np": True},
312-
{"sources": ["_sgd_fast.pyx"], "include_np": True},
312+
{"sources": ["_sgd_fast.pyx.tp"], "include_np": True},
313313
{"sources": ["_sag_fast.pyx.tp"], "include_np": True},
314314
],
315315
"manifold": [

‎sklearn/linear_model/_sgd_fast.pyx renamed to ‎sklearn/linear_model/_sgd_fast.pyx.tp

Lines changed: 108 additions & 47 deletions
F438
Original file line numberDiff line numberDiff line change
@@ -1,22 +1,50 @@
1-
# Author: Peter Prettenhofer <peter.prettenhofer@gmail.com>
2-
# Mathieu Blondel (partial_fit support)
3-
# Rob Zinkov (passive-aggressive)
4-
# Lars Buitinck
5-
#
6-
# License: BSD 3 clause
1+
{{py:
72

3+
"""
4+
Template file to easily generate fused types consistent code using Tempita
5+
(https://github.com/cython/cython/blob/master/Cython/Tempita/_tempita.py).
86

7+
Generated file: _sgd_fast.pyx
8+
9+
Each relevant function is duplicated for the dtypes float and double.
10+
The keywords between double braces are substituted in setup.py.
11+
12+
Authors: Peter Prettenhofer <peter.prettenhofer@gmail.com>
13+
Mathieu Blondel (partial_fit support)
14+
Rob Zinkov (passive-aggressive)
15+
Lars Buitinck
16+
17+
License: BSD 3 clause
18+
"""
19+
20+
# The dtypes are defined as follows (name_suffix, c_type, np_type)
21+
dtypes = [
22+
("64", "double", "np.float64"),
23+
("32", "float", "np.float32"),
24+
]
25+
26+
}}
27+
28+
#------------------------------------------------------------------------------
29+
30+
"""
31+
SGD implementation
32+
WARNING: Do not edit .pyx file directly, it is generated from .pyx.tp
33+
"""
34+
35+
from cython cimport floating
936
import numpy as np
1037
from time import time
1138

1239
from libc.math cimport exp, log, pow, fabs
1340
cimport numpy as cnp
1441
from numpy.math cimport INFINITY
1542
cdef extern from "_sgd_fast_helpers.h":
16-
bint skl_isfinite(double) nogil
43+
bint skl_isfinite32(float) nogil
44+
bint skl_isfinite64(double) nogil
1745

18-
from ..utils._weight_vector cimport WeightVector64 as WeightVector
19-
from ..utils._seq_dataset cimport SequentialDataset64 as SequentialDataset
46+
from ..utils._weight_vector cimport WeightVector32, WeightVector64
47+
from ..utils._seq_dataset cimport SequentialDataset32, SequentialDataset64
2048

2149
cnp.import_array()
2250

@@ -372,37 +400,48 @@ cdef class SquaredEpsilonInsensitive(Regression):
372400
def __reduce__(self):
373401
return SquaredEpsilonInsensitive, (self.epsilon,)
374402

375-
376-
def _plain_sgd(const double[::1] weights,
377-
double intercept,
378-
const double[::1] average_weights,
379-
double average_intercept,
380-
LossFunction loss,
381-
int penalty_type,
382-
double alpha, double C,
383-
double l1_ratio,
384-
SequentialDataset dataset,
385-
const unsigned char[::1] validation_mask,
386-
bint early_stopping, validation_score_cb,
387-
int n_iter_no_change,
388-
unsigned int max_iter, double tol, int fit_intercept,
389-
int verbose, bint shuffle, cnp.uint32_t seed,
390-
double weight_pos, double weight_neg,
391-
int learning_rate, double eta0,
392-
double power_t,
393-
bint one_class,
394-
double t=1.0,
395-
double intercept_decay=1.0,
396-
int average=0):
403+
{{for name_suffix, c_type, np_type in dtypes}}
404+
405+
def _plain_sgd{{name_suffix}}(
406+
const {{c_type}}[::1] weights,
407+
double intercept,
408+
const {{c_type}}[::1] average_weights,
409+
double average_intercept,
410+
LossFunction loss,
411+
int penalty_type,
412+
double alpha,
413+
double C,
414+
double l1_ratio,
415+
SequentialDataset{{name_suffix}} dataset,
416+
const unsigned char[::1] validation_mask,
417+
bint early_stopping,
418+
validation_score_cb,
419+
int n_iter_no_change,
420+
unsigned int max_iter,
421+
double tol,
422+
int fit_intercept,
423+
int verbose,
424+
bint shuffle,
425+
cnp.uint32_t seed,
426+
double weight_pos,
427+
double weight_neg,
428+
int learning_rate,
429+
double eta0,
430+
double power_t,
431+
bint one_class,
432+
double t=1.0,
433+
double intercept_decay=1.0,
434+
int average=0,
435+
):
397436
"""SGD for generic loss functions and penalties with optional averaging
398437

399438
Parameters
400439
----------
401-
weights : ndarray[double, ndim=1]
440+
weights : ndarray[{{c_type}}, ndim=1]
402441
The allocated vector of weights.
403442
intercept : double
404443
The initial intercept.
405-
average_weights : ndarray[double, ndim=1]
444+
average_weights : ndarray[{{c_type}}, ndim=1]
406445
The average weights as computed for ASGD. Should be None if average
407446
is 0.
408447
average_intercept : double
@@ -489,8 +528,8 @@ def _plain_sgd(const double[::1] weights,
489528
cdef Py_ssize_t n_samples = dataset.n_samples
490529
cdef Py_ssize_t n_features = weights.shape[0]
491530

492-
cdef WeightVector w = WeightVector(weights, average_weights)
493-
cdef double *x_data_ptr = NULL
531+
cdef WeightVector{{name_suffix}} w = WeightVector{{name_suffix}}(weights, average_weights)
532+
cdef {{c_type}} *x_data_ptr = NULL
494533
cdef int *x_ind_ptr = NULL
495534

496535
# helper variables
@@ -505,9 +544,9 @@ def _plain_sgd(const double[::1] weights,
505544
cdef double score = 0.0
506545
cdef double best_loss = INFINITY
507546
cdef double best_score = -INFINITY
508-
cdef double y = 0.0
509-
cdef double sample_weight
510-
cdef double class_weight = 1.0
547+
cdef {{c_type}} y = 0.0
548+
cdef {{c_type}} sample_weight
549+
cdef {{c_type}} class_weight = 1.0
511550
cdef unsigned int count = 0
512551
cdef unsigned int train_count = n_samples - np.sum(validation_mask)
513552
cdef unsigned int epoch = 0
@@ -520,10 +559,10 @@ def _plain_sgd(const double[::1] weights,
520559
cdef long long sample_index
521560

522561
# q vector is only used for L1 regularization
523-
cdef double[::1] q = None
524-
cdef double * q_data_ptr = NULL
562+
cdef {{c_type}}[::1] q = None
563+
cdef {{c_type}} * q_data_ptr = NULL
525564
if penalty_type == L1 or penalty_type == ELASTICNET:
526-
q = np.zeros((n_features,), dtype=np.float64, order="c")
565+
q = np.zeros((n_features,), dtype={{np_type}}, order="c")
527566
q_data_ptr = &q[0]
528567
cdef double u = 0.0
529568

@@ -627,7 +666,7 @@ def _plain_sgd(const double[::1] weights,
627666

628667
if penalty_type == L1 or penalty_type == ELASTICNET:
629668
u += (l1_ratio * eta * alpha)
630-
l1penalty(w, q_data_ptr, x_ind_ptr, xnnz, u)
669+
l1penalty{{name_suffix}}(w, q_data_ptr, x_ind_ptr, xnnz, u)
631670

632671
t += 1
633672
count += 1
@@ -694,15 +733,28 @@ def _plain_sgd(const double[::1] weights,
694733
epoch + 1
695734
)
696735

736+
{{endfor}}
737+
738+
739+
cdef inline bint skl_isfinite(floating w) noexcept nogil:
740+
if floating is float:
741+
return skl_isfinite32(w)
742+
else:
743+
return skl_isfinite64(w)
697744

698-
cdef bint any_nonfinite(const double *w, int n) noexcept nogil:
745+
746+
cdef inline bint any_nonfinite(const floating *w, int n) noexcept nogil:
699747
for i in range(n):
700748
if not skl_isfinite(w[i]):
701749
return True
702750
return 0
703751

704752

705-
cdef double sqnorm(double * x_data_ptr, int * x_ind_ptr, int xnnz) noexcept nogil:
753+
cdef inline double sqnorm(
754+
floating * x_data_ptr,
755+
int * x_ind_ptr,
756+
int xnnz,
757+
) noexcept nogil:
706758
cdef double x_norm = 0.0
707759
cdef int j
708760
cdef double z
@@ -712,8 +764,15 @@ cdef double sqnorm(double * x_data_ptr, int * x_ind_ptr, int xnnz) noexcept nogi
712764
return x_norm
713765

714766

715-
cdef void l1penalty(WeightVector w, double * q_data_ptr,
716-
int *x_ind_ptr, int xnnz, double u) noexcept nogil:
767+
{{for name_suffix, c_type, np_type in dtypes}}
768+
769+
cdef void l1penalty{{name_suffix}}(
770+
WeightVector{{name_suffix}} w,
771+
{{c_type}} * q_data_ptr,
772+
int *x_ind_ptr,
773+
int xnnz,
774+
double u,
775+
) noexcept nogil:
717776
"""Apply the L1 penalty to each updated feature
718777

719778
This implements the truncated gradient approach by
@@ -723,7 +782,7 @@ cdef void l1penalty(WeightVector w, double * q_data_ptr,
723782
cdef int j = 0
724783
cdef int idx = 0
725784
cdef double wscale = w.wscale
726-
cdef double *w_data_ptr = w.w_data_ptr
785+
cdef {{c_type}} *w_data_ptr = w.w_data_ptr
727786
for j in range(xnnz):
728787
idx = x_ind_ptr[j]
729788
z = w_data_ptr[idx]
@@ -736,3 +795,5 @@ cdef void l1penalty(WeightVector w, double * q_data_ptr,
736795
0.0, w_data_ptr[idx] + ((u - q_data_ptr[idx]) / wscale))
737796

738797
q_data_ptr[idx] += wscale * (w_data_ptr[idx] - z)
798+
799+
{{endfor}}

0 commit comments

Comments
 (0)
0