From 916c6ce45c6979083803587e82330cfc796a968c Mon Sep 17 00:00:00 2001 From: jeremie du boisberranger Date: Mon, 21 Oct 2019 14:11:42 +0200 Subject: [PATCH 1/6] lazy import of tempita to avoid cython dependency when building from c files --- sklearn/utils/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sklearn/utils/setup.py b/sklearn/utils/setup.py index 370bd20e01d69..f7478bd38e2d8 100644 --- a/sklearn/utils/setup.py +++ b/sklearn/utils/setup.py @@ -5,7 +5,6 @@ def configuration(parent_package='', top_path=None): import numpy from numpy.distutils.misc_util import Configuration - from Cython import Tempita config = Configuration('utils', parent_package, top_path) @@ -58,6 +57,7 @@ def configuration(parent_package='', top_path=None): with open(pyxfiles, "r") as f: tmpl = f.read() + from Cython import Tempita # noqa pyxcontent = Tempita.sub(tmpl) with open(outfile, "w") as f: From 47b2dea7f51acfe08bd2290f6a09003081d7ea97 Mon Sep 17 00:00:00 2001 From: jeremie du boisberranger Date: Tue, 22 Oct 2019 11:00:37 +0200 Subject: [PATCH 2/6] utility function --- sklearn/_build_utils/__init__.py | 23 +++++++++++++++++++++++ sklearn/linear_model/setup.py | 18 ++++-------------- sklearn/utils/setup.py | 24 ++++++------------------ 3 files changed, 33 insertions(+), 32 deletions(-) diff --git a/sklearn/_build_utils/__init__.py b/sklearn/_build_utils/__init__.py index a2e69c6978efe..09545508ffd89 100644 --- a/sklearn/_build_utils/__init__.py +++ b/sklearn/_build_utils/__init__.py @@ -75,3 +75,26 @@ def maybe_cythonize_extensions(top_path, config): nthreads=n_jobs, compile_time_env={'SKLEARN_OPENMP_SUPPORTED': with_openmp}, compiler_directives={'language_level': 3}) + + +def gen_from_templates(templates): + """Generate cython files from templates""" + if not isinstance(templates, list): + templates = [templates] + + for template in templates: + pyxfile = template.replace('.tp', '') + + if not (os.path.exists(pyxfile) and + os.stat(template).st_mtime < os.stat(pyxfile).st_mtime): + + with open(template, "r") as f: + tmpl = f.read() + + # Lazy import because cython is not a dependency when building from + # source distribution. + from Cython import Tempita # noqa + tmpl_ = Tempita.sub(tmpl) + + with open(pyxfile, "w") as f: + f.write(tmpl_) diff --git a/sklearn/linear_model/setup.py b/sklearn/linear_model/setup.py index 8226412fdecbd..f174ec5d063ed 100644 --- a/sklearn/linear_model/setup.py +++ b/sklearn/linear_model/setup.py @@ -1,7 +1,9 @@ import os - import numpy +from .._build_utils import gen_from_templates + + def configuration(parent_package='', top_path=None): from numpy.distutils.misc_util import Configuration @@ -22,19 +24,7 @@ def configuration(parent_package='', top_path=None): libraries=libraries) # generate sag_fast from template - sag_cython_file = 'sklearn/linear_model/sag_fast.pyx.tp' - sag_file = sag_cython_file.replace('.tp', '') - - if not (os.path.exists(sag_file) and - os.stat(sag_cython_file).st_mtime < os.stat(sag_file).st_mtime): - - with open(sag_cython_file, "r") as f: - tmpl = f.read() - from Cython import Tempita # noqa - tmpl_ = Tempita.sub(tmpl) - - with open(sag_file, "w") as f: - f.write(tmpl_) + gen_from_templates('sklearn/linear_model/sag_fast.pyx.tp') config.add_extension('sag_fast', sources=['sag_fast.pyx'], diff --git a/sklearn/utils/setup.py b/sklearn/utils/setup.py index f7478bd38e2d8..dbeccef2de052 100644 --- a/sklearn/utils/setup.py +++ b/sklearn/utils/setup.py @@ -1,6 +1,8 @@ import os from os.path import join +from .._build_utils import gen_from_templates + def configuration(parent_package='', top_path=None): import numpy @@ -44,24 +46,10 @@ def configuration(parent_package='', top_path=None): sources=['_openmp_helpers.pyx'], libraries=libraries) - # generate files from a template - pyx_templates = ['sklearn/utils/_seq_dataset.pyx.tp', - 'sklearn/utils/_seq_dataset.pxd.tp'] - - for pyxfiles in pyx_templates: - outfile = pyxfiles.replace('.tp', '') - # if .pyx.tp is not updated, no need to output .pyx - if (os.path.exists(outfile) and - os.stat(pyxfiles).st_mtime < os.stat(outfile).st_mtime): - continue - - with open(pyxfiles, "r") as f: - tmpl = f.read() - from Cython import Tempita # noqa - pyxcontent = Tempita.sub(tmpl) - - with open(outfile, "w") as f: - f.write(pyxcontent) + # generate _seq_dataset from template + templates = ['sklearn/utils/_seq_dataset.pyx.tp', + 'sklearn/utils/_seq_dataset.pxd.tp'] + gen_from_templates(templates) config.add_extension('_seq_dataset', sources=['_seq_dataset.pyx'], From 31a58ed8fe4968cb096eec556df375677249b8ad Mon Sep 17 00:00:00 2001 From: jeremie du boisberranger Date: Tue, 22 Oct 2019 13:17:42 +0200 Subject: [PATCH 3/6] trigger ci From a36a37d4f94798f6ff9cc3b0640d915b656e1866 Mon Sep 17 00:00:00 2001 From: jeremie du boisberranger Date: Tue, 22 Oct 2019 13:30:54 +0200 Subject: [PATCH 4/6] fix abs import --- sklearn/linear_model/setup.py | 2 +- sklearn/utils/setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/sklearn/linear_model/setup.py b/sklearn/linear_model/setup.py index f174ec5d063ed..b2570e68c9032 100644 --- a/sklearn/linear_model/setup.py +++ b/sklearn/linear_model/setup.py @@ -1,7 +1,7 @@ import os import numpy -from .._build_utils import gen_from_templates +from sklearn._build_utils import gen_from_templates def configuration(parent_package='', top_path=None): diff --git a/sklearn/utils/setup.py b/sklearn/utils/setup.py index dbeccef2de052..f62021992aee5 100644 --- a/sklearn/utils/setup.py +++ b/sklearn/utils/setup.py @@ -1,7 +1,7 @@ import os from os.path import join -from .._build_utils import gen_from_templates +from sklearn._build_utils import gen_from_templates def configuration(parent_package='', top_path=None): From 4ad3d74425cb59a8f84707c6aa0970937b6f75d8 Mon Sep 17 00:00:00 2001 From: jeremie du boisberranger Date: Wed, 23 Oct 2019 17:44:05 +0200 Subject: [PATCH 5/6] rename --- sklearn/_build_utils/__init__.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/sklearn/_build_utils/__init__.py b/sklearn/_build_utils/__init__.py index 09545508ffd89..8ceabf62946f8 100644 --- a/sklearn/_build_utils/__init__.py +++ b/sklearn/_build_utils/__init__.py @@ -83,10 +83,10 @@ def gen_from_templates(templates): templates = [templates] for template in templates: - pyxfile = template.replace('.tp', '') + outfile = template.replace('.tp', '') - if not (os.path.exists(pyxfile) and - os.stat(template).st_mtime < os.stat(pyxfile).st_mtime): + if not (os.path.exists(outfile) and + os.stat(template).st_mtime < os.stat(outfile).st_mtime): with open(template, "r") as f: tmpl = f.read() @@ -96,5 +96,5 @@ def gen_from_templates(templates): from Cython import Tempita # noqa tmpl_ = Tempita.sub(tmpl) - with open(pyxfile, "w") as f: + with open(outfile, "w") as f: f.write(tmpl_) From 1f6f6129ab20424c3345d3b75942520851717223 Mon Sep 17 00:00:00 2001 From: jeremie du boisberranger Date: Thu, 24 Oct 2019 14:04:30 +0200 Subject: [PATCH 6/6] clear distinction for sdist case --- sklearn/_build_utils/__init__.py | 18 +++++++++++------- sklearn/linear_model/setup.py | 3 ++- sklearn/utils/setup.py | 2 +- 3 files changed, 14 insertions(+), 9 deletions(-) diff --git a/sklearn/_build_utils/__init__.py b/sklearn/_build_utils/__init__.py index 8ceabf62946f8..1d6c1eaf607a6 100644 --- a/sklearn/_build_utils/__init__.py +++ b/sklearn/_build_utils/__init__.py @@ -77,23 +77,27 @@ def maybe_cythonize_extensions(top_path, config): compiler_directives={'language_level': 3}) -def gen_from_templates(templates): - """Generate cython files from templates""" - if not isinstance(templates, list): - templates = [templates] +def gen_from_templates(templates, top_path): + """Generate cython files from a list of templates""" + is_release = os.path.exists(os.path.join(top_path, 'PKG-INFO')) + # Files are already cythonized, nothing to do. + if is_release: + return + + # Lazy import because cython is not a dependency when building from + # source distribution. + from Cython import Tempita # noqa for template in templates: outfile = template.replace('.tp', '') + # if the template is not updated, no need to output the cython file if not (os.path.exists(outfile) and os.stat(template).st_mtime < os.stat(outfile).st_mtime): with open(template, "r") as f: tmpl = f.read() - # Lazy import because cython is not a dependency when building from - # source distribution. - from Cython import Tempita # noqa tmpl_ = Tempita.sub(tmpl) with open(outfile, "w") as f: diff --git a/sklearn/linear_model/setup.py b/sklearn/linear_model/setup.py index 6bec913b3b2fc..121b449d673d0 100644 --- a/sklearn/linear_model/setup.py +++ b/sklearn/linear_model/setup.py @@ -24,7 +24,8 @@ def configuration(parent_package='', top_path=None): libraries=libraries) # generate sag_fast from template - gen_from_templates('sklearn/linear_model/_sag_fast.pyx.tp') + templates = ['sklearn/linear_model/_sag_fast.pyx.tp'] + gen_from_templates(templates, top_path) config.add_extension('_sag_fast', sources=['_sag_fast.pyx'], diff --git a/sklearn/utils/setup.py b/sklearn/utils/setup.py index f62021992aee5..098adeeccab09 100644 --- a/sklearn/utils/setup.py +++ b/sklearn/utils/setup.py @@ -49,7 +49,7 @@ def configuration(parent_package='', top_path=None): # generate _seq_dataset from template templates = ['sklearn/utils/_seq_dataset.pyx.tp', 'sklearn/utils/_seq_dataset.pxd.tp'] - gen_from_templates(templates) + gen_from_templates(templates, top_path) config.add_extension('_seq_dataset', sources=['_seq_dataset.pyx'],