8000 Rewriting of cythonization in setup.py · scikit-learn/scikit-learn@1eade7d · GitHub
[go: up one dir, main page]
Skip to content

Commit 1eade7d

Browse files
lestevelesteve
committed
Rewriting of cythonization in setup.py
By using Cython.Build.cythonize and switching between .c and .pyx files as appropriate cython dependencies are correctly taken into account.
1 parent 61a9a9b commit 1eade7d

File tree

19 files changed

+191
-279
lines changed

19 files changed

+191
-279
lines changed

‎build_tools/cythonize.py

Lines changed: 0 additions & 198 deletions
This file was deleted.

‎setup.py

Lines changed: 1 addition & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -84,9 +84,6 @@ def run(self):
8484
cwd = os.path.abspath(os.path.dirname(__file__))
8585
remove_c_files = not os.path.exists(os.path.join(cwd, 'PKG-INFO'))
8686
if remove_c_files:
87-
cython_hash_file = os.path.join(cwd, 'cythonize.dat')
88-
if os.path.exists(cython_hash_file):
89-
os.unlink(cython_hash_file)
9087
print('Will remove generated .c files')
9188
if os.path.exists('build'):
9289
shutil.rmtree('build')
@@ -181,18 +178,6 @@ def get_numpy_status():
181178
return numpy_status
182179

183180

184-
def generate_cython():
185-
cwd = os.path.abspath(os.path.dirname(__file__))
186-
print("Cythonizing sources")
187-
p = subprocess.call([sys.executable, os.path.join(cwd,
188-
'build_tools',
189-
'cythonize.py'),
190-
'sklearn'],
191-
cwd=cwd)
192-
if p != 0:
193-
raise RuntimeError("Running cythonize failed!")
194-
195-
196181
def setup_package():
197182
metadata = dict(name=DISTNAME,
198183
maintainer=MAINTAINER,
@@ -230,7 +215,7 @@ def setup_package():
230215
'egg_info',
231216
'--version',
232217
'clean'))):
233-
# For these actions, NumPy is not required, nor Cythonization
218+
# For these actions, NumPy is not required
234219
#
235220
# They are required to succeed without Numpy for example when
236221
# pip is used to install Scikit-learn when Numpy is not yet present in
@@ -278,26 +263,6 @@ def setup_package():
278263

279264
metadata['configuration'] = configuration
280265

281-
if len(sys.argv) >= 2 and sys.argv[1] not in 'config':
282-
# Cythonize if needed
283-
284-
print('Generating cython files')
285-
cwd = os.path.abspath(os.path.dirname(__file__))
286-
if not os.path.exists(os.path.join(cwd, 'PKG-INFO')):
287-
# Generate Cython sources, unless building from source release
288-
generate_cython()
289-
290-
# Clean left-over .so file
291-
for dirpath, dirnames, filenames in os.walk(
292-
os.path.join(cwd, 'sklearn')):
293-
for filename in filenames:
294-
extension = os.path.splitext(filename)[1]
295-
if extension in (".so", ".pyd", ".dll"):
296-
pyx_file = str.replace(filename, extension, '.pyx')
297-
print(pyx_file)
298-
if not os.path.exists(os.path.join(dirpath, pyx_file)):
299-
os.unlink(os.path.join(dirpath, filename))
300-
301266
setup(**metadata)
302267

303268

‎sklearn/__check_build/setup.py

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,11 +3,15 @@
33

44
import numpy
55

6+
from sklearn._build_utils import add_cython_extension
7+
68

79
def configuration(parent_package='', top_path=None):
810
from numpy.distutils.misc_util import Configuration
911
config = Configuration('__check_build', parent_package, top_path)
10-
config.add_extension('_check_build',
12+
add_cython_extension(top_path,
13+
config,
14+
'_check_build',
1115
sources=['_check_build.c'],
1216
include_dirs=[numpy.get_include()])
1317

‎sklearn/_build_utils/__init__.py

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,8 @@
66

77
from __future__ import division, print_function, absolute_import
88

9+
import os
10+
911
DEFAULT_ROOT = 'sklearn'
1012

1113
from numpy.distutils.system_info import get_info
@@ -33,3 +35,38 @@ def atlas_not_found(blas_info_):
3335
cblas_libs = blas_info.pop('libraries', [])
3436

3537
return cblas_libs, blas_info
38+
39+
40+
def get_cython_source(filename):
41+
is_c_filename = filename.endswith('.c')
42+
is_cpp_filename = filename.endswith('.cpp')
43+
44+
# files in src are .c and .cpp files that are not cython-generated
45+
if 'src/' in filename and (is_c_filename or is_cpp_filename):
46+
return filename
47+
elif is_c_filename:
48+
filename = filename[:-1]
49+
elif is_cpp_filename:
50+
filename = filename[:-3]
51+
else:
52+
raise ValueError('Only .c and .cpp files are supported. '
53+
'Got {0!r} instead'.format(filename))
54+
return filename + 'pyx'
55+
56+
57+
def add_cython_extension(top_path, config, name, sources, **kwargs):
58+
is_dev_version = not os.path.exists(os.path.join(top_path, 'PKG-INFO'))
59+
60+
if is_dev_version:
61+
try:
62+
from Cython.Build import cythonize
63+
except ImportError:
64+
raise ValueError('Please install cython in order '
65+
'to build a scikit-learn development version')
66+
67+
sources = [get_cython_source(filename) for filename in sources]
68+
69+
config.add_extension(name, sources, **kwargs)
70+
71+
if is_dev_version:
72+
config.ext_modules = cythonize(config.ext_modules)

‎sklearn/cluster/setup.py

Lines changed: 13 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66
import numpy
77

88
from sklearn._build_utils import get_blas_info
9+
from sklearn._build_utils import add_cython_extension
910

1011

1112
def configuration(parent_package='', top_path=None):
@@ -19,22 +20,30 @@ def configuration(parent_package='', top_path=None):
1920
libraries.append('m')
2021

2122
config = Configuration('cluster', parent_package, top_path)
22-
config.add_extension('_dbscan_inner',
23+
add_cython_extension(top_path,
24+
config,
25+
'_dbscan_inner',
2326
sources=['_dbscan_inner.cpp'],
2427
include_dirs=[numpy.get_include()],
2528
language="c++")
2629

27-
config.add_extension('_hierarchical',
30+
add_cython_extension(top_path,
31+
config,
32+
'_hierarchical',
2833
sources=['_hierarchical.cpp'],
2934
language="c++",
3035
include_dirs=[numpy.get_include()],
3136
libraries=libraries)
32-
config.add_extension('_k_means_elkan',
37+
add_cython_extension(top_path,
38+
config,
39+
'_k_means_elkan',
3340
sources=['_k_means_elkan.c'],
3441
include_dirs=[numpy.get_include()],
3542
libraries=libraries)
3643

37-
config.add_extension(
44+
add_cython_extension(
45+
top_path,
46+
config,
3847
'_k_means',
3948
libraries=cblas_libs,
4049
sources=['_k_means.c'],

‎sklearn/datasets/setup.py

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,8 @@
22
import numpy
33
import os
44

5+
from sklearn._build_utils import add_cython_extension
6+
57

68
def configuration(parent_package='', top_path=None):
79
from numpy.distutils.misc_util import Configuration
@@ -10,7 +12,9 @@ def configuration(parent_package='', top_path=None):
1012
config.add_data_dir('descr')
1113
config.add_data_dir('images')
1214
config.add_data_dir(os.path.join('tests', 'data'))
13-
config.add_extension('_svmlight_format',
15+
add_cython_extension(top_path,
16+
config,
17+
'_svmlight_format',
1418
sources=['_svmlight_format.c'],
1519
include_dirs=[numpy.get_include()])
1620
config.add_subpackage('tests')

0 commit comments

Comments
 (0)
0