diff --git a/MANIFEST.in b/MANIFEST.in
index 37ab0f7d..fb3f4ff5 100644
--- a/MANIFEST.in
+++ b/MANIFEST.in
@@ -2,4 +2,4 @@ include MANIFEST.in
 include README.rst
 include Makefile
 include requirements*.txt
-recursive-include lightning *.c *.h *.cpp *.pyx *.pxd
+recursive-include lightning *.c *.h *.pyx *.pxd
diff --git a/lightning/_build_utils.py b/lightning/_build_utils.py
deleted file mode 100644
index cacd292b..00000000
--- a/lightning/_build_utils.py
+++ /dev/null
@@ -1,59 +0,0 @@
-"""
-Utilities useful during the build.
-"""
-# author: Loic Esteve
-# license: BSD
-import os
-
-from distutils.version import LooseVersion
-
-DEFAULT_ROOT = 'sklearn'
-CYTHON_MIN_VERSION = '0.23'
-
-try:
-    from sklearn._build_utils import maybe_cythonize_extensions
-except ImportError:
-    # maybe_cythonize_extensions exists from scikit-learn 0.18.1 onwards
-    def build_from_c_and_cpp_files(extensions):
-        """Modify the extensions to build from the .c and .cpp files.
-
-        This is useful for releases, this way cython is not required to
-        run python setup.py install.
-        """
-        for extension in extensions:
-            sources = []
-            for sfile in extension.sources:
-                path, ext = os.path.splitext(sfile)
-                if ext in ('.pyx', '.py'):
-                    if extension.language == 'c++':
-                        ext = '.cpp'
-                    else:
-                        ext = '.c'
-                    sfile = path + ext
-                sources.append(sfile)
-            extension.sources = sources
-
-
-    def maybe_cythonize_extensions(top_path, config):
-        """Tweaks for building extensions between release and development mode."""
-        is_release = os.path.exists(os.path.join(top_path, 'PKG-INFO'))
-
-        if is_release:
-            build_from_c_and_cpp_files(config.ext_modules)
-        else:
-            message = ('Please install cython with a version >= {0} in order '
-                       'to build a scikit-learn development version.').format(
-                           CYTHON_MIN_VERSION)
-            try:
-                import Cython
-                if LooseVersion(Cython.__version__) < CYTHON_MIN_VERSION:
-                    message += ' Your version of Cython was {0}.'.format(
-                        Cython.__version__)
-                    raise ValueError(message)
-                from Cython.Build import cythonize
-            except ImportError as exc:
-                exc.args += (message,)
-                raise
-
-            config.ext_modules = cythonize(config.ext_modules)
-
diff --git a/lightning/setup.py b/lightning/setup.py
index 7161f2e7..0aa753c3 100644
--- a/lightning/setup.py
+++ b/lightning/setup.py
@@ -1,7 +1,18 @@
+import sys
+
 from numpy.distutils.core import setup
 from numpy.distutils.misc_util import Configuration
 
-from lightning._build_utils import maybe_cythonize_extensions
+
+def cythonize_extensions(top_path, config):
+    try:
+        from Cython.Build import cythonize
+    except ModuleNotFoundError as e:
+        raise ModuleNotFoundError(
+            'Please install Cython in order to build a lightning from source.') from e
+
+    config.ext_modules = cythonize(config.ext_modules,
+                                   compiler_directives={'language_level': 3})
 
 
 def configuration(parent_package='', top_path=None):
@@ -9,7 +20,11 @@ def configuration(parent_package='', top_path=None):
 
     config.add_subpackage('impl')
 
-    maybe_cythonize_extensions(top_path, config)
+    # Skip cythonization as we do not want to include the generated
+    # C/C++ files in the release tarballs as they are not necessarily
+    # forward compatible with future versions of Python for instance.
+    if 'sdist' not in sys.argv:
+        cythonize_extensions(top_path, config)
 
     return config
 
diff --git a/setup.py b/setup.py
index 8b468b8f..a21cc291 100644
--- a/setup.py
+++ b/setup.py
@@ -5,9 +5,12 @@
 import re
 import sys
 import os
-from distutils.command.sdist import sdist
 import setuptools
+
+from distutils.command.sdist import sdist
+
 from numpy.distutils.core import setup
+from numpy.distutils.misc_util import Configuration
 
 
 DISTNAME = 'sklearn-contrib-lightning'
@@ -19,7 +22,6 @@
 MAINTAINER_EMAIL = 'mathieu@mblondel.org'
 URL = 'https://github.com/scikit-learn-contrib/lightning'
 LICENSE = 'new BSD'
-DOWNLOAD_URL = URL
 with open(os.path.join('lightning', '__init__.py'), encoding='utf-8') as f:
     match = re.search(r'__version__[ ]*=[ ]*[\"\'](?P<version>.+)[\"\']',
                       f.read())
@@ -37,9 +39,15 @@ def configuration(parent_package='', top_path=None):
     if os.path.exists('MANIFEST'):
         os.remove('MANIFEST')
 
-    from numpy.distutils.misc_util import Configuration
     config = Configuration(None, parent_package, top_path)
 
+    # Avoid non-useful msg:
+    # "Ignoring attempt to set 'name' (from ... "
+    config.set_options(ignore_setup_xxx_py=True,
+                       assume_default_configuration=True,
+                       delegate_options_to_subpackages=True,
+                       quiet=True)
+
     config.add_subpackage('lightning')
 
     return config
@@ -55,7 +63,7 @@ def configuration(parent_package='', top_path=None):
     setup(configuration=configuration,
           name=DISTNAME,
           maintainer=MAINTAINER,
-          python_requires='>={}'.format(MIN_PYTHON_VERSION),
+          python_requires=f'>={MIN_PYTHON_VERSION}',
           install_requires=REQUIREMENTS,
           include_package_data=True,
           scripts=["bin/lightning_train",
@@ -65,7 +73,7 @@ def configuration(parent_package='', top_path=None):
           license=LICENSE,
           url=URL,
           version=VERSION,
-          download_url=DOWNLOAD_URL,
+          download_url=URL,
           long_description=LONG_DESCRIPTION,
           zip_safe=False,  # the package can run out of an .egg file
           cmdclass={"sdist": sdist},
@@ -74,7 +82,7 @@ def configuration(parent_package='', top_path=None):
               'Intended Audience :: Developers',
               'License :: OSI Approved',
               'Programming Language :: C',
-              'Programming Language :: Python',
+              'Programming Language :: Python :: 3',
               'Topic :: Software Development',
               'Topic :: Scientific/Engineering',
               'Operating System :: Microsoft :: Windows',