From 6772684f9acb733c0356681b368396a52a851ba6 Mon Sep 17 00:00:00 2001 From: jeremie du boisberranger Date: Thu, 14 Nov 2019 17:15:46 +0100 Subject: [PATCH 1/5] include all pxd files in package --- setup.py | 1 + sklearn/tree/setup.py | 4 ---- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/setup.py b/setup.py index 3ed5c786a17c3..80b1b8ecc9391 100755 --- a/setup.py +++ b/setup.py @@ -259,6 +259,7 @@ def setup_package(): 'scipy>={}'.format(SCIPY_MIN_VERSION), 'joblib>={}'.format(JOBLIB_MIN_VERSION) ], + package_data={'': ['*.pxd']}, **extra_setuptools_args) if len(sys.argv) == 1 or ( diff --git a/sklearn/tree/setup.py b/sklearn/tree/setup.py index 2b9819795b74b..079ae9d869075 100644 --- a/sklearn/tree/setup.py +++ b/sklearn/tree/setup.py @@ -31,10 +31,6 @@ def configuration(parent_package="", top_path=None): extra_compile_args=["-O3"]) config.add_subpackage("tests") - config.add_data_files("_criterion.pxd") - config.add_data_files("_splitter.pxd") - config.add_data_files("_tree.pxd") - config.add_data_files("_utils.pxd") return config From 4c86cd84f69f386f1ca2ccf690cf0c407f020ba3 Mon Sep 17 00:00:00 2001 From: jeremie du boisberranger Date: Wed, 20 Nov 2019 17:42:05 +0100 Subject: [PATCH 2/5] _libsvm _liblinear to pxi + add a test --- maint_tools/check_pxd_in_installation.py | 72 +++++++++++++++++++ .../svm/{_liblinear.pxd => _liblinear.pxi} | 5 +- sklearn/svm/_liblinear.pyx | 2 + sklearn/svm/{_libsvm.pxd => _libsvm.pxi} | 2 - sklearn/svm/_libsvm.pyx | 2 + sklearn/utils/_weight_vector.pxd | 7 -- 6 files changed, 78 insertions(+), 12 deletions(-) create mode 100644 maint_tools/check_pxd_in_installation.py rename sklearn/svm/{_liblinear.pxd => _liblinear.pxi} (98%) rename sklearn/svm/{_libsvm.pxd => _libsvm.pxi} (99%) diff --git a/maint_tools/check_pxd_in_installation.py b/maint_tools/check_pxd_in_installation.py new file mode 100644 index 0000000000000..210278d91af17 --- /dev/null +++ b/maint_tools/check_pxd_in_installation.py @@ -0,0 +1,72 @@ +"""Utility for testing presence and usability of .pxd files in the installation + +Usage: +------ +python check_pxd_in_installation.py path/to/install_dir/of/scikit-learn +""" + +import os +import sys +import glob +import tempfile +import textwrap +import subprocess + + +skl_dir = sys.argv[1] +skl_dir = os.path.abspath(skl_dir) + +# find all .pxd files +pxd_files = glob.glob(os.path.join(skl_dir, '**', '*.pxd'), recursive=True) +pxd_files = [f.replace(skl_dir, 'sklearn') for f in pxd_files] + +print("> Found pxd files:") +for file in pxd_files: + print(' -', file) + +print("\n> Trying to compile a cython extension cimporting all corresponding " + "modules\n") +start_dir = os.path.abspath('.') + +with tempfile.TemporaryDirectory() as tmpdir: + try: + os.chdir(tmpdir) + + # A cython test file which cimports all modules corresponding to found + # pxd files. + # e.g. sklearn/tree/_utils.pxd becomes `cimport sklearn.tree._utils` + with open('tst.pyx', 'w') as f: + for file in pxd_files: + to_import = file.replace(os.path.sep, '.') + to_import = to_import.replace('.pxd', '') + f.write('cimport ' + to_import + '\n') + + # A basic setup file to build the test file. + # We set the language to c++ and we use numpy.get_include() because + # some modules require it. + with open('setup_tst.py', 'w') as f: + f.write(textwrap.dedent( + """ + from distutils.core import setup + from distutils.extension import Extension + from Cython.Build import cythonize + import numpy + + extensions = [Extension("tst", + sources=["tst.pyx"], + language="c++", + include_dirs=[numpy.get_include()])] + + setup(ext_modules=cythonize(extensions)) + """)) + + subprocess.run(["python", "setup_tst.py", "build_ext", "-i"], + check=True) + + except subprocess.CalledProcessError: + raise + + finally: + os.chdir(start_dir) + + print("\n> Compilation succeeded !") diff --git a/sklearn/svm/_liblinear.pxd b/sklearn/svm/_liblinear.pxi similarity index 98% rename from sklearn/svm/_liblinear.pxd rename to sklearn/svm/_liblinear.pxi index 0f10e54a532fe..148bf694dab4f 100644 --- a/sklearn/svm/_liblinear.pxd +++ b/sklearn/svm/_liblinear.pxi @@ -1,6 +1,3 @@ -cimport numpy as np - - cdef extern from "_cython_blas_helpers.h": ctypedef double (*dot_func)(int, double*, int, double*, int) ctypedef void (*axpy_func)(int, double, double*, int, double*, int) @@ -12,6 +9,7 @@ cdef extern from "_cython_blas_helpers.h": scal_func scal nrm2_func nrm2 + cdef extern from "linear.h": cdef struct feature_node cdef struct problem @@ -28,6 +26,7 @@ cdef extern from "linear.h": void free_and_destroy_model (model **) void destroy_param (parameter *) + cdef extern from "liblinear_helper.c": void copy_w(void *, model *, int) parameter *set_parameter(int, double, double, int, char *, char *, int, int, double) diff --git a/sklearn/svm/_liblinear.pyx b/sklearn/svm/_liblinear.pyx index 2f042748d94a0..9dd15e0716c7f 100644 --- a/sklearn/svm/_liblinear.pyx +++ b/sklearn/svm/_liblinear.pyx @@ -9,6 +9,8 @@ cimport numpy as np from ..utils._cython_blas cimport _dot, _axpy, _scal, _nrm2 +include "_liblinear.pxi" + np.import_array() diff --git a/sklearn/svm/_libsvm.pxd b/sklearn/svm/_libsvm.pxi similarity index 99% rename from sklearn/svm/_libsvm.pxd rename to sklearn/svm/_libsvm.pxi index 2664a335a372f..a3c8f1c33dd1e 100644 --- a/sklearn/svm/_libsvm.pxd +++ b/sklearn/svm/_libsvm.pxi @@ -1,5 +1,3 @@ -cimport numpy as np - ################################################################################ # Includes diff --git a/sklearn/svm/_libsvm.pyx b/sklearn/svm/_libsvm.pyx index 8f8e9f7465823..079a791fef3b6 100644 --- a/sklearn/svm/_libsvm.pyx +++ b/sklearn/svm/_libsvm.pyx @@ -35,6 +35,8 @@ import numpy as np cimport numpy as np from libc.stdlib cimport free +include "_libsvm.pxi" + cdef extern from *: ctypedef struct svm_parameter: pass diff --git a/sklearn/utils/_weight_vector.pxd b/sklearn/utils/_weight_vector.pxd index 1f38bb7e0981f..fc1b47a50ef1f 100644 --- a/sklearn/utils/_weight_vector.pxd +++ b/sklearn/utils/_weight_vector.pxd @@ -1,12 +1,5 @@ """Efficient (dense) parameter vector implementation for linear models. """ -cimport numpy as np - - -cdef extern from "math.h": - cdef extern double sqrt(double x) - - cdef class WeightVector(object): cdef double *w_data_ptr cdef double *aw_data_ptr From 6d713dadcf92381fd52e9f236e53f3988111800e Mon Sep 17 00:00:00 2001 From: jeremie du boisberranger Date: Fri, 22 Nov 2019 17:21:52 +0100 Subject: [PATCH 3/5] simplify stuff + use pathlib --- maint_tools/check_pxd_in_installation.py | 84 +++++++++++------------- 1 file changed, 38 insertions(+), 46 deletions(-) diff --git a/maint_tools/check_pxd_in_installation.py b/maint_tools/check_pxd_in_installation.py index 210278d91af17..5d6f1c3fba3a4 100644 --- a/maint_tools/check_pxd_in_installation.py +++ b/maint_tools/check_pxd_in_installation.py @@ -7,66 +7,58 @@ import os import sys -import glob +import pathlib import tempfile import textwrap import subprocess -skl_dir = sys.argv[1] -skl_dir = os.path.abspath(skl_dir) +sklearn_dir = pathlib.Path(sys.argv[1]).absolute() +print(sklearn_dir) # find all .pxd files -pxd_files = glob.glob(os.path.join(skl_dir, '**', '*.pxd'), recursive=True) -pxd_files = [f.replace(skl_dir, 'sklearn') for f in pxd_files] +pxd_files = list(sklearn_dir.glob("**/*.pxd")) print("> Found pxd files:") -for file in pxd_files: - print(' -', file) +for pxd_file in pxd_files: + print(' -', pxd_file) print("\n> Trying to compile a cython extension cimporting all corresponding " "modules\n") start_dir = os.path.abspath('.') with tempfile.TemporaryDirectory() as tmpdir: - try: - os.chdir(tmpdir) - - # A cython test file which cimports all modules corresponding to found - # pxd files. - # e.g. sklearn/tree/_utils.pxd becomes `cimport sklearn.tree._utils` - with open('tst.pyx', 'w') as f: - for file in pxd_files: - to_import = file.replace(os.path.sep, '.') - to_import = to_import.replace('.pxd', '') - f.write('cimport ' + to_import + '\n') - - # A basic setup file to build the test file. - # We set the language to c++ and we use numpy.get_include() because - # some modules require it. - with open('setup_tst.py', 'w') as f: - f.write(textwrap.dedent( - """ - from distutils.core import setup - from distutils.extension import Extension - from Cython.Build import cythonize - import numpy - - extensions = [Extension("tst", - sources=["tst.pyx"], - language="c++", - include_dirs=[numpy.get_include()])] - - setup(ext_modules=cythonize(extensions)) - """)) - - subprocess.run(["python", "setup_tst.py", "build_ext", "-i"], - check=True) - - except subprocess.CalledProcessError: - raise - - finally: - os.chdir(start_dir) + tmpdir = pathlib.Path(tmpdir) + # A cython test file which cimports all modules corresponding to found + # pxd files. + # e.g. sklearn/tree/_utils.pxd becomes `cimport sklearn.tree._utils` + with open(tmpdir / 'tst.pyx', 'w') as f: + for pxd_file in pxd_files: + to_import = str(pxd_file.relative_to(sklearn_dir)) + to_import = to_import.replace(os.path.sep, '.') + to_import = to_import.replace('.pxd', '') + f.write('cimport sklearn.' + to_import + '\n') + + # A basic setup file to build the test file. + # We set the language to c++ and we use numpy.get_include() because + # some modules require it. + with open(tmpdir / 'setup_tst.py', 'w') as f: + f.write(textwrap.dedent( + """ + from distutils.core import setup + from distutils.extension import Extension + from Cython.Build import cythonize + import numpy + + extensions = [Extension("tst", + sources=["tst.pyx"], + language="c++", + include_dirs=[numpy.get_include()])] + + setup(ext_modules=cythonize(extensions)) + """)) + + subprocess.run(["python", "setup_tst.py", "build_ext", "-i"], + check=True, cwd=tmpdir) print("\n> Compilation succeeded !") From 7eb03555f649aab71a351ba6d1ee9e1938c4f0be Mon Sep 17 00:00:00 2001 From: jeremie du boisberranger Date: Fri, 22 Nov 2019 17:23:44 +0100 Subject: [PATCH 4/5] cln --- maint_tools/check_pxd_in_installation.py | 1 - 1 file changed, 1 deletion(-) diff --git a/maint_tools/check_pxd_in_installation.py b/maint_tools/check_pxd_in_installation.py index 5d6f1c3fba3a4..9fd04f620552e 100644 --- a/maint_tools/check_pxd_in_installation.py +++ b/maint_tools/check_pxd_in_installation.py @@ -14,7 +14,6 @@ sklearn_dir = pathlib.Path(sys.argv[1]).absolute() -print(sklearn_dir) # find all .pxd files pxd_files = list(sklearn_dir.glob("**/*.pxd")) From a0d204661b4767f10caad0e6673d0d1c260ea9b0 Mon Sep 17 00:00:00 2001 From: jeremie du boisberranger Date: Fri, 22 Nov 2019 17:26:31 +0100 Subject: [PATCH 5/5] more cln --- maint_tools/check_pxd_in_installation.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/maint_tools/check_pxd_in_installation.py b/maint_tools/check_pxd_in_installation.py index 9fd04f620552e..83c4b706294ad 100644 --- a/maint_tools/check_pxd_in_installation.py +++ b/maint_tools/check_pxd_in_installation.py @@ -13,9 +13,7 @@ import subprocess -sklearn_dir = pathlib.Path(sys.argv[1]).absolute() - -# find all .pxd files +sklearn_dir = pathlib.Path(sys.argv[1]) pxd_files = list(sklearn_dir.glob("**/*.pxd")) print("> Found pxd files:") @@ -24,8 +22,6 @@ print("\n> Trying to compile a cython extension cimporting all corresponding " "modules\n") -start_dir = os.path.abspath('.') - with tempfile.TemporaryDirectory() as tmpdir: tmpdir = pathlib.Path(tmpdir) # A cython test file which cimports all modules corresponding to found