From fe4928122cd35a06cf2c120bb0360e23022b6563 Mon Sep 17 00:00:00 2001 From: Namami Shanker Date: Mon, 11 Jul 2022 20:40:43 +0530 Subject: [PATCH 01/37] FIX: Import f2py2e rather than f2py for run_main --- numpy/distutils/command/build_src.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/numpy/distutils/command/build_src.py b/numpy/distutils/command/build_src.py index bf3d03c70e44..7786b949a160 100644 --- a/numpy/distutils/command/build_src.py +++ b/numpy/distutils/command/build_src.py @@ -558,8 +558,8 @@ def f2py_sources(self, sources, extension): and not skip_f2py: log.info("f2py:> %s" % (target_file)) self.mkpath(target_dir) - import numpy.f2py - numpy.f2py.run_main(f2py_options + ['--lower', + from numpy.f2py import f2py2e + f2py2e.run_main(f2py_options + ['--lower', '--build-dir', target_dir]+\ ['-m', ext_name]+f_sources) else: From 6b1475e17cb2e84910fb64358058cc098a80d6c2 Mon Sep 17 00:00:00 2001 From: Namami Shanker Date: Tue, 12 Jul 2022 19:31:20 +0530 Subject: [PATCH 02/37] FIX: Import f2py2e instead of f2py --- numpy/distutils/command/build_src.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/numpy/distutils/command/build_src.py b/numpy/distutils/command/build_src.py index 7786b949a160..7303db124cc8 100644 --- a/numpy/distutils/command/build_src.py +++ b/numpy/distutils/command/build_src.py @@ -539,8 +539,8 @@ def f2py_sources(self, sources, extension): if (self.force or newer_group(depends, target_file, 'newer')) \ and not skip_f2py: log.info("f2py: %s" % (source)) - import numpy.f2py - numpy.f2py.run_main(f2py_options + from numpy.f2py import f2py2e + f2py2e.run_main(f2py_options + ['--build-dir', target_dir, source]) else: log.debug(" skipping '%s' f2py interface (up-to-date)" % (source)) From 91f3825346dea3537f2bf1ce9fb4ca2b97b173ce Mon Sep 17 00:00:00 2001 From: Rohit Goswami Date: Thu, 24 Aug 2023 22:44:55 +0000 Subject: [PATCH 03/37] ENH: Add F2PY back-end work from gh-22225 Co-authored-by: NamamiShanker --- numpy/f2py/backends/__init__.py | 4 + numpy/f2py/backends/backend.py | 125 +++++++++++ numpy/f2py/backends/meson_backend.py | 310 +++++++++++++++++++++++++++ 3 files changed, 439 insertions(+) create mode 100644 numpy/f2py/backends/__init__.py create mode 100644 numpy/f2py/backends/backend.py create mode 100644 numpy/f2py/backends/meson_backend.py diff --git a/numpy/f2py/backends/__init__.py b/numpy/f2py/backends/__init__.py new file mode 100644 index 000000000000..7c93fa16a715 --- /dev/null +++ b/numpy/f2py/backends/__init__.py @@ -0,0 +1,4 @@ +from .backend import Backend +from .meson_backend import MesonBackend + +backends = {"meson": MesonBackend} diff --git a/numpy/f2py/backends/backend.py b/numpy/f2py/backends/backend.py new file mode 100644 index 000000000000..30f0046c9f3e --- /dev/null +++ b/numpy/f2py/backends/backend.py @@ -0,0 +1,125 @@ +from __future__ import annotations + +from pathlib import Path +from abc import ABC, abstractmethod + +import numpy +import numpy.f2py as f2py + + +class Backend(ABC): + """ + Superclass for backend compilation plugins to extend. + + """ + + def __init__( + self, + module_name, + fortran_compiler: str, + c_compiler: str, + f77exec: Path, + f90exec: Path, + f77_flags: list[str], + f90_flags: list[str], + include_paths: list[Path], + include_dirs: list[Path], + external_resources: list[str], + linker_libpath: list[Path], + linker_libname: list[str], + define_macros: list[tuple[str, str]], + undef_macros: list[str], + debug: bool, + opt_flags: list[str], + arch_flags: list[str], + no_opt: bool, + no_arch: bool, + ) -> None: + """ + The class is initialized with f2py compile options. + The parameters are mappings of f2py compilation flags. + + Parameters + ---------- + module_name : str + The name of the module to be compiled. (-m) + fortran_compiler : str + Name of the Fortran compiler to use. (--fcompiler) + c_compiler : str + Name of the C compiler to use. (--ccompiler) + f77exec : Pathlike + Path to the fortran compiler for Fortran 77 files (--f77exec) + f90exec : Pathlike + Path to the fortran compiler for Fortran 90 and above files (--f90exec) + f77_flags : list + List of flags to pass to the fortran compiler for Fortran 77 files (--f77flags) + f90_flags : list + List of flags to pass to the fortran compiler for Fortran 90 and above files (--f90flags) + include_paths : list + Search include files from given directories (--include-paths) + include_dirs : list + Append directory to the list of directories searched for include files. (-I) + external_resources : list + Link the extension module with (--link-) + linker_libname : list + Use the library when linking. (-l) + define_macros : list + Define to if present else define to true (-D) + undef_macros : list + Undefine (-U) + linker_libpath : list + Add directory to the list of directories to be searched for `-l`. (-L) + opt_flags : list + Optimization flags to pass to the compiler. (--opt) + arch_flags : list + Architectire specific flags to pass to the compiler (--arch) + no_opt : bool + Disable optimization. (--no-opt) + no_arch : bool + Disable architecture specific optimizations. (--no-arch) + debug : bool + Enable debugging. (--debug) + + """ + self.module_name = module_name + self.fortran_compiler = fortran_compiler + self.c_compiler = c_compiler + self.f77exec = f77exec + self.f90exec = f90exec + self.f77_flags = f77_flags + self.f90_flags = f90_flags + self.include_paths = include_paths + self.include_dirs = include_dirs + self.external_resources = external_resources + self.linker_libpath = linker_libpath + self.linker_libname = linker_libname + self.define_macros = define_macros + self.undef_macros = undef_macros + self.debug = debug + self.opt_flags = opt_flags + self.arch_flags = arch_flags + self.no_opt = no_opt + self.no_arch = no_arch + + def numpy_install_path(self) -> Path: + """ + Returns the install path for numpy. + """ + return Path(numpy.__file__).parent + + def numpy_get_include(self) -> Path: + """ + Returns the include paths for numpy. + """ + return Path(numpy.get_include()) + + def f2py_get_include(self) -> Path: + """ + Returns the include paths for f2py. + """ + return Path(f2py.get_include()) + + @abstractmethod + def compile(self, fortran_sources: Path, c_wrapper: Path, build_dir: Path) -> None: + """Compile the wrapper.""" + pass diff --git a/numpy/f2py/backends/meson_backend.py b/numpy/f2py/backends/meson_backend.py new file mode 100644 index 000000000000..df0d1b4fdb8b --- /dev/null +++ b/numpy/f2py/backends/meson_backend.py @@ -0,0 +1,310 @@ +from __future__ import annotations + +import os +import errno +import shutil +import subprocess +from pathlib import Path + +from .backend import Backend +from string import Template + + +class MesonTemplate: + """Template meson build file generation class.""" + + def __init__( + self, + module_name: str, + numpy_install_path, + numpy_get_include: Path, + f2py_get_include: Path, + wrappers: list[Path], + fortran_sources: list[Path], + dependencies: list[str], + include_path: list[Path], + optimization_flags: list[str], + architecture_flags: list[str], + f77_flags: list[str], + f90_flags: list[str], + linker_libpath: list[Path], + linker_libname: list[str], + define_macros: list[tuple[str, str]], + undef_macros: list[str], + ): + self.module_name = module_name + self.numpy_install_path = numpy_install_path + self.build_template_path = ( + numpy_install_path / "f2py" / "backends" / "src" / "meson.build.src" + ) + self.sources = fortran_sources + self.numpy_get_include = numpy_get_include + self.f2py_get_include = f2py_get_include + self.wrappers = wrappers + self.dependencies = dependencies + self.include_directories = include_path + self.substitutions = {} + self.optimization_flags = optimization_flags + self.architecture_flags = architecture_flags + self.fortran_flags = f77_flags + f90_flags + self.linker_libpath = linker_libpath + self.linker_libname = linker_libname + self.define_macros = define_macros + self.undef_macros = undef_macros + self.pipeline = [ + self.initialize_template, + self.global_flags_substitution, + self.sources_substitution, + self.dependencies_substitution, + self.include_directories_subtitution, + self.linker_substitution, + self.macros_substitution, + ] + + @property + def meson_build_template(self) -> str: + if not self.build_template_path.is_file(): + raise FileNotFoundError( + errno.ENOENT, + f"Meson build template {self.build_template_path.absolute()} does not exist.", + ) + return self.build_template_path.read_text() + + def initialize_template(self) -> None: + """Initialize with module name and external NumPy and F2PY C libraries.""" + self.substitutions["modulename"] = self.module_name + self.substitutions["numpy_get_include"] = self.numpy_get_include.absolute() + self.substitutions["f2py_get_include"] = self.f2py_get_include.absolute() + + def sources_substitution(self) -> None: + self.substitutions["source_list"] = ",".join( + ["'" + str(source.absolute()) + "'" for source in self.sources] + ) + self.substitutions["wrappers"] = ",".join( + ["'" + str(wrapper.absolute()) + "'" for wrapper in self.wrappers] + ) + + def dependencies_substitution(self) -> None: + self.substitutions["dependencies_list"] = ", ".join( + [f"dependency('{dependecy}')" for dependecy in self.dependencies] + ) + + def include_directories_subtitution(self) -> None: + self.substitutions["include_directories_list"] = ", ".join( + [ + f"include_directories('{include_directory}')" + for include_directory in self.include_directories + ] + ) + + def global_flags_substitution(self) -> None: + fortran_compiler_flags = ( + self.fortran_flags + self.optimization_flags + self.architecture_flags + ) + c_compiler_flags = self.optimization_flags + self.architecture_flags + self.substitutions["fortran_global_args"] = fortran_compiler_flags + self.substitutions["c_global_args"] = c_compiler_flags + + def macros_substitution(self) -> None: + self.substitutions["macros"] = "" + if self.define_macros: + self.substitutions["macros"] = ",".join( + f"'-D{macro[0]}={macro[1]}'" if macro[1] else f"-D{macro[0]}" + for macro in self.define_macros + ) + if self.undef_macros: + self.substitutions["macros"] += "," + ",".join( + f"'-U{macro}'" for macro in self.undef_macros + ) + + def linker_substitution(self) -> None: + self.substitutions["linker_args"] = "" + if self.linker_libpath: + linker_libpath_subs = ",".join( + f"-L{libpath}" for libpath in self.linker_libpath + ) + self.substitutions["linker_args"] += linker_libpath_subs + if self.linker_libname: + linker_libname_subs = ",".join( + f"-l{libname}" for libname in self.linker_libname + ) + self.substitutions["linker_args"] += f",{linker_libname_subs}" + + def generate_meson_build(self) -> str: + for node in self.pipeline: + node() + template = Template(self.meson_build_template) + return template.substitute(self.substitutions) + + +class MesonBackend(Backend): + def __init__( + self, + module_name: str = "untitled", + fortran_compiler: str = None, + c_compiler: str = None, + f77exec: Path = None, + f90exec: Path = None, + f77_flags: list[str] = None, + f90_flags: list[str] = None, + include_paths: list[Path] = None, + include_dirs: list[Path] = None, + external_resources: list[str] = None, + linker_libpath: list[Path] = None, + linker_libname: list[str] = None, + define_macros: list[tuple[str, str]] = None, + undef_macros: list[str] = None, + debug: bool = False, + opt_flags: list[str] = None, + arch_flags: list[str] = None, + no_opt: bool = False, + no_arch: bool = False, + ) -> None: + self.meson_build_dir = "builddir" + if f77_flags is None: + f77_flags = [] + if include_paths is None: + include_paths = [] + if include_dirs is None: + include_dirs = [] + if external_resources is None: + external_resources = [] + if linker_libpath is None: + linker_libpath = [] + if linker_libname is None: + linker_libname = [] + if define_macros is None: + define_macros = [] + if undef_macros is None: + undef_macros = [] + if f77_flags is None: + f77_flags = [] + if f90_flags is None: + f90_flags = [] + if opt_flags is None: + opt_flags = [] + if arch_flags is None: + arch_flags = [] + super().__init__( + module_name, + fortran_compiler, + c_compiler, + f77exec, + f90exec, + f77_flags, + f90_flags, + include_paths, + include_dirs, + external_resources, + linker_libpath, + linker_libname, + define_macros, + undef_macros, + debug, + opt_flags, + arch_flags, + no_opt, + no_arch, + ) + + self.wrappers: list[Path] = [] + self.fortran_sources: list[Path] = [] + self.template = Template(self.fortran_sources) + + def _get_optimization_level(self): + if self.no_arch and not self.no_opt: + return 2 + elif self.no_opt: + return 0 + return 3 + + def _set_environment_variables(self) -> None: + if self.fortran_compiler: + os.putenv("FC", self.fortran_compiler) + elif self.f77exec: + os.putenv("FC", self.f77exec) + elif self.f90exec: + os.putenv("FC", self.f90exec) + if self.c_compiler: + os.putenv("CC", self.c_compiler) + + def _move_exec_to_root(self, build_dir: Path): + walk_dir = build_dir / self.meson_build_dir + path_objects = walk_dir.glob(f"{self.module_name}*.so") + for path_object in path_objects: + shutil.move(path_object, Path.cwd()) + + def _get_build_command(self): + return [ + "meson", + "setup", + self.meson_build_dir, + "-Ddebug=true" if self.debug else "-Ddebug=false", + f"-Doptimization={str(self._get_optimization_level())}", + ] + + def load_wrapper(self, wrappers: list[Path]) -> None: + self.wrappers = wrappers + + def load_sources(self, fortran_sources: list[Path]) -> None: + for fortran_source in fortran_sources: + fortran_source = Path(fortran_source) + if not fortran_source.is_file(): + raise FileNotFoundError( + errno.ENOENT, f"{fortran_source.absolute()} does not exist." + ) + self.fortran_sources.append(fortran_source) + + def write_meson_build(self, build_dir: Path) -> None: + """Writes the meson build file at specified location""" + meson_template = MesonTemplate( + self.module_name, + super().numpy_install_path(), + self.numpy_get_include(), + self.f2py_get_include(), + self.wrappers, + self.fortran_sources, + self.external_resources, + self.include_paths + self.include_dirs, + optimization_flags=self.opt_flags, + architecture_flags=self.arch_flags, + f77_flags=self.f77_flags, + f90_flags=self.f90_flags, + linker_libpath=self.linker_libpath, + linker_libname=self.linker_libname, + define_macros=self.define_macros, + undef_macros=self.undef_macros, + ) + src = meson_template.generate_meson_build() + meson_build_file = build_dir / "meson.build" + meson_build_file.write_text(src) + return meson_build_file + + def run_meson(self, build_dir: Path): + self._set_environment_variables() + completed_process = subprocess.run(self._get_build_command(), cwd=build_dir) + if completed_process.returncode != 0: + raise subprocess.CalledProcessError( + completed_process.returncode, completed_process.args + ) + completed_process = subprocess.run( + ["meson", "compile", "-C", self.meson_build_dir], cwd=build_dir + ) + if completed_process.returncode != 0: + raise subprocess.CalledProcessError( + completed_process.returncode, completed_process.args + ) + + def compile( + self, + f77_sources: list[Path], + f90_sources: list[Path], + object_files: list[Path], + wrappers: list[Path], + build_dir: Path, + ) -> None: + self.load_wrapper(wrappers) + self.load_sources(f77_sources + f90_sources + object_files) + self.write_meson_build(build_dir) + self.run_meson(build_dir) + self._move_exec_to_root(build_dir) From a0b01c38872b71d901aa281676bd17547d4f1579 Mon Sep 17 00:00:00 2001 From: Rohit Goswami Date: Thu, 24 Aug 2023 22:46:56 +0000 Subject: [PATCH 04/37] ENH: Add meson skeleton from gh-2225 Co-authored-by: NamamiShanker --- numpy/f2py/backends/src/meson.build.src | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 numpy/f2py/backends/src/meson.build.src diff --git a/numpy/f2py/backends/src/meson.build.src b/numpy/f2py/backends/src/meson.build.src new file mode 100644 index 000000000000..8d2ce5fde994 --- /dev/null +++ b/numpy/f2py/backends/src/meson.build.src @@ -0,0 +1,22 @@ +project('${modulename}', 'c', + version : '0.1', + default_options : ['warning_level=2']) + +add_languages('fortran') + +add_global_arguments(${c_global_args}, language: 'c') +add_global_arguments(${fortran_global_args}, language: 'fortran') + +py_mod = import('python') +py3 = py_mod.find_installation('python3') +py3_dep = py3.dependency() +message(py3.path()) +message(py3.get_install_dir()) + +incdir_numpy = '${numpy_get_include}' + +incdir_f2py = '${f2py_get_include}' + +inc_np = include_directories(incdir_numpy, incdir_f2py) + +py3.extension_module('${modulename}', ${wrappers}, ${source_list}, incdir_f2py+'/fortranobject.c', include_directories: [inc_np, ${include_directories_list}], dependencies : [py3_dep, ${dependencies_list}], link_args : '${linker_args}', c_args : [${macros}], install : true) From 384aa9640c51d4ddf23b195e389eee442eded62b Mon Sep 17 00:00:00 2001 From: Rohit Goswami Date: Fri, 25 Aug 2023 00:49:08 +0000 Subject: [PATCH 05/37] MAINT: Trim backend.py down to f2py2e flags --- numpy/f2py/backends/backend.py | 133 ++++++--------------------------- 1 file changed, 23 insertions(+), 110 deletions(-) diff --git a/numpy/f2py/backends/backend.py b/numpy/f2py/backends/backend.py index 30f0046c9f3e..d86789d1915f 100644 --- a/numpy/f2py/backends/backend.py +++ b/numpy/f2py/backends/backend.py @@ -1,125 +1,38 @@ from __future__ import annotations -from pathlib import Path from abc import ABC, abstractmethod -import numpy -import numpy.f2py as f2py - class Backend(ABC): - """ - Superclass for backend compilation plugins to extend. - - """ - def __init__( self, - module_name, - fortran_compiler: str, - c_compiler: str, - f77exec: Path, - f90exec: Path, - f77_flags: list[str], - f90_flags: list[str], - include_paths: list[Path], - include_dirs: list[Path], - external_resources: list[str], - linker_libpath: list[Path], - linker_libname: list[str], - define_macros: list[tuple[str, str]], - undef_macros: list[str], - debug: bool, - opt_flags: list[str], - arch_flags: list[str], - no_opt: bool, - no_arch: bool, - ) -> None: - """ - The class is initialized with f2py compile options. - The parameters are mappings of f2py compilation flags. - - Parameters - ---------- - module_name : str - The name of the module to be compiled. (-m) - fortran_compiler : str - Name of the Fortran compiler to use. (--fcompiler) - c_compiler : str - Name of the C compiler to use. (--ccompiler) - f77exec : Pathlike - Path to the fortran compiler for Fortran 77 files (--f77exec) - f90exec : Pathlike - Path to the fortran compiler for Fortran 90 and above files (--f90exec) - f77_flags : list - List of flags to pass to the fortran compiler for Fortran 77 files (--f77flags) - f90_flags : list - List of flags to pass to the fortran compiler for Fortran 90 and above files (--f90flags) - include_paths : list - Search include files from given directories (--include-paths) - include_dirs : list - Append directory to the list of directories searched for include files. (-I) - external_resources : list - Link the extension module with (--link-) - linker_libname : list - Use the library when linking. (-l) - define_macros : list - Define to if present else define to true (-D) - undef_macros : list - Undefine (-U) - linker_libpath : list - Add directory to the list of directories to be searched for `-l`. (-L) - opt_flags : list - Optimization flags to pass to the compiler. (--opt) - arch_flags : list - Architectire specific flags to pass to the compiler (--arch) - no_opt : bool - Disable optimization. (--no-opt) - no_arch : bool - Disable architecture specific optimizations. (--no-arch) - debug : bool - Enable debugging. (--debug) - - """ - self.module_name = module_name - self.fortran_compiler = fortran_compiler - self.c_compiler = c_compiler - self.f77exec = f77exec - self.f90exec = f90exec - self.f77_flags = f77_flags - self.f90_flags = f90_flags - self.include_paths = include_paths + modulename, + include_dirs, + library_dirs, + libraries, + define_macros, + undef_macros, + f2py_flags, + sysinfo_flags, + fc_flags, + flib_flags, + setup_flags, + remove_build_dir, + ): + self.modulename = modulename self.include_dirs = include_dirs - self.external_resources = external_resources - self.linker_libpath = linker_libpath - self.linker_libname = linker_libname + self.library_dirs = library_dirs + self.libraries = libraries self.define_macros = define_macros self.undef_macros = undef_macros - self.debug = debug - self.opt_flags = opt_flags - self.arch_flags = arch_flags - self.no_opt = no_opt - self.no_arch = no_arch - - def numpy_install_path(self) -> Path: - """ - Returns the install path for numpy. - """ - return Path(numpy.__file__).parent - - def numpy_get_include(self) -> Path: - """ - Returns the include paths for numpy. - """ - return Path(numpy.get_include()) - - def f2py_get_include(self) -> Path: - """ - Returns the include paths for f2py. - """ - return Path(f2py.get_include()) + self.f2py_flags = f2py_flags + self.sysinfo_flags = sysinfo_flags + self.fc_flags = fc_flags + self.flib_flags = flib_flags + self.setup_flags = setup_flags + self.remove_build_dir = remove_build_dir @abstractmethod - def compile(self, fortran_sources: Path, c_wrapper: Path, build_dir: Path) -> None: + def compile(self, sources, extra_objects, build_dir) -> None: """Compile the wrapper.""" pass From 26b224f49e4bab8057196162c70ef7a18d5d53e3 Mon Sep 17 00:00:00 2001 From: Rohit Goswami Date: Fri, 25 Aug 2023 00:49:45 +0000 Subject: [PATCH 06/37] ENH: Add a factory function for backends --- numpy/f2py/backends/__init__.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/numpy/f2py/backends/__init__.py b/numpy/f2py/backends/__init__.py index 7c93fa16a715..cd3a6a93f002 100644 --- a/numpy/f2py/backends/__init__.py +++ b/numpy/f2py/backends/__init__.py @@ -1,4 +1,11 @@ -from .backend import Backend -from .meson_backend import MesonBackend +def get_backend(name): + if name == "meson": + from .meson_backend import MesonBackend -backends = {"meson": MesonBackend} + return MesonBackend + elif name == "distutils": + from .distutils_backend import DistutilsBackend + + return DistutilsBackend + else: + raise ValueError(f"Unknown backend: {name}") From 3a5c43d2dc15ab482f438e56c72423a164cf5923 Mon Sep 17 00:00:00 2001 From: Rohit Goswami Date: Fri, 25 Aug 2023 00:50:00 +0000 Subject: [PATCH 07/37] ENH: Add a distutils backend --- numpy/f2py/backends/distutils_backend.py | 71 ++++++++++++++++++++++++ 1 file changed, 71 insertions(+) create mode 100644 numpy/f2py/backends/distutils_backend.py diff --git a/numpy/f2py/backends/distutils_backend.py b/numpy/f2py/backends/distutils_backend.py new file mode 100644 index 000000000000..83bd8558e5db --- /dev/null +++ b/numpy/f2py/backends/distutils_backend.py @@ -0,0 +1,71 @@ +from numpy.f2py.backends.backend import Backend + +from numpy.distutils.core import setup, Extension +from numpy.distutils.system_info import get_info +from numpy.distutils.misc_util import dict_append +import os +import sys +import shutil +import warnings + + +class DistutilsBackend(Backend): + def __init__(sef, *args, **kwargs): + warnings.warn( + "distutils has been deprecated since NumPy 2.16." + "Use the MesonBackend instead, or generate wrappers" + "without -c and use a custom build script", + np.VisibleDeprecationWarning, stacklevel=2) + super().__init__(*args, **kwargs) + + def compile(self, sources, extra_objects, build_dir): + num_info = {} + if num_info: + self.include_dirs.extend(num_info.get("include_dirs", [])) + ext_args = { + "name": self.modulename, + "sources": sources, + "include_dirs": self.include_dirs, + "library_dirs": self.library_dirs, + "libraries": self.libraries, + "define_macros": self.define_macros, + "undef_macros": self.undef_macros, + "extra_objects": extra_objects, + "f2py_options": self.f2py_flags, + } + + if self.sysinfo_flags: + for n in self.sysinfo_flags: + i = get_info(n) + if not i: + print( + f"No {repr(n)} resources found in system (try `f2py --help-link`)" + ) + dict_append(ext_args, **i) + + ext = Extension(**ext_args) + + sys.argv = [sys.argv[0]] + self.setup_flags + sys.argv.extend( + [ + "build", + "--build-temp", + build_dir, + "--build-base", + build_dir, + "--build-platlib", + ".", + "--disable-optimization", + ] + ) + + if self.fc_flags: + sys.argv.extend(["config_fc"] + self.fc_flags) + if self.flib_flags: + sys.argv.extend(["build_ext"] + self.flib_flags) + + setup(ext_modules=[ext]) + + if self.remove_build_dir and os.path.exists(build_dir): + print(f"Removing build directory {build_dir}") + shutil.rmtree(build_dir) From d03d28de31a1f82df2fc2a75702b7e4838d2c0f7 Mon Sep 17 00:00:00 2001 From: Rohit Goswami Date: Fri, 25 Aug 2023 00:50:30 +0000 Subject: [PATCH 08/37] ENH: Handle --backends in f2py Defaults to distutils for now --- numpy/f2py/f2py2e.py | 54 ++++++++++++-------------------------------- 1 file changed, 15 insertions(+), 39 deletions(-) diff --git a/numpy/f2py/f2py2e.py b/numpy/f2py/f2py2e.py index 10508488dc04..0b72e5d52058 100755 --- a/numpy/f2py/f2py2e.py +++ b/numpy/f2py/f2py2e.py @@ -28,6 +28,7 @@ from . import f90mod_rules from . import __version__ from . import capi_maps +from . import backends f2py_version = __version__.version numpy_version = __version__.version @@ -588,6 +589,15 @@ def run_compile(): if '--quiet' in f2py_flags: setup_flags.append('--quiet') + # Check for and remove --backend first + if '--backend' in sys.argv: + backend_index = sys.argv.index('--backend') + backend_key = sys.argv.pop(backend_index + 1) + sys.argv.pop(backend_index) + else: + backend_key = 'distutils' + build_backend = backends.get_backend(backend_key) + modulename = 'untitled' sources = sys.argv[1:] @@ -633,46 +643,12 @@ def run_compile(): if num_info: include_dirs.extend(num_info.get('include_dirs', [])) - from numpy.distutils.core import setup, Extension - ext_args = {'name': modulename, 'sources': sources, - 'include_dirs': include_dirs, - 'library_dirs': library_dirs, - 'libraries': libraries, - 'define_macros': define_macros, - 'undef_macros': undef_macros, - 'extra_objects': extra_objects, - 'f2py_options': f2py_flags, - } - - if sysinfo_flags: - from numpy.distutils.misc_util import dict_append - for n in sysinfo_flags: - i = get_info(n) - if not i: - outmess('No %s resources found in system' - ' (try `f2py --help-link`)\n' % (repr(n))) - dict_append(ext_args, **i) - - ext = Extension(**ext_args) - sys.argv = [sys.argv[0]] + setup_flags - sys.argv.extend(['build', - '--build-temp', build_dir, - '--build-base', build_dir, - '--build-platlib', '.', - # disable CCompilerOpt - '--disable-optimization']) - if fc_flags: - sys.argv.extend(['config_fc'] + fc_flags) - if flib_flags: - sys.argv.extend(['build_ext'] + flib_flags) - - setup(ext_modules=[ext]) - - if remove_build_dir and os.path.exists(build_dir): - import shutil - outmess('Removing build directory %s\n' % (build_dir)) - shutil.rmtree(build_dir) + # Now use the builder + builder = build_backend(modulename, include_dirs, library_dirs, libraries, + define_macros, undef_macros, f2py_flags, + sysinfo_flags, fc_flags, flib_flags, setup_flags, remove_build_dir) + builder.compile(sources, extra_objects, build_dir) def main(): if '--help-link' in sys.argv[1:]: From 1d279bc106ed6b9d9c3bc1e5980649404b383f1c Mon Sep 17 00:00:00 2001 From: Rohit Goswami Date: Fri, 25 Aug 2023 00:50:55 +0000 Subject: [PATCH 09/37] DOC: Add some minor comments in f2py2e --- numpy/f2py/f2py2e.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/numpy/f2py/f2py2e.py b/numpy/f2py/f2py2e.py index 0b72e5d52058..60fc2913dacd 100755 --- a/numpy/f2py/f2py2e.py +++ b/numpy/f2py/f2py2e.py @@ -252,6 +252,8 @@ def scaninputline(inputline): 'f2py option --include_paths is deprecated, use --include-paths instead.\n') f7 = 1 elif l[:15] in '--include-paths': + # Similar to using -I with -c, however this is + # also used during generation of wrappers f7 = 1 elif l == '--skip-empty-wrappers': emptygen = False @@ -675,6 +677,12 @@ def main(): pass if '-c' in sys.argv[1:]: + import warnings + warnings.warn('Using f2py to compile code without generating files ' + 'first was largely handled by distutils which has been removed in Python 3.12.' + 'The meson replacement is a work in progress, consider running without -c.' + 'Recall that the build can be customized with --build-dir', + stacklevel=2) run_compile() else: run_main(sys.argv[1:]) From 0f0e3ac4cf8e84de4321833e0d1b9df506cfe264 Mon Sep 17 00:00:00 2001 From: Rohit Goswami Date: Fri, 25 Aug 2023 01:05:19 +0000 Subject: [PATCH 10/37] MAINT: Refactor and rework meson.build.src --- numpy/f2py/backends/src/meson.build.src | 38 +++++++++++++++++++------ 1 file changed, 29 insertions(+), 9 deletions(-) diff --git a/numpy/f2py/backends/src/meson.build.src b/numpy/f2py/backends/src/meson.build.src index 8d2ce5fde994..b1f87818ed7d 100644 --- a/numpy/f2py/backends/src/meson.build.src +++ b/numpy/f2py/backends/src/meson.build.src @@ -1,22 +1,42 @@ project('${modulename}', 'c', version : '0.1', + meson_version: '>= 1.1.0', default_options : ['warning_level=2']) add_languages('fortran') -add_global_arguments(${c_global_args}, language: 'c') -add_global_arguments(${fortran_global_args}, language: 'fortran') +add_project_arguments(${c_global_args}, language: 'c') +add_project_arguments(${fortran_global_args}, language: 'fortran') py_mod = import('python') -py3 = py_mod.find_installation('python3') -py3_dep = py3.dependency() -message(py3.path()) -message(py3.get_install_dir()) +py = py_mod.find_installation(pure: false) +py_dep = py.dependency() -incdir_numpy = '${numpy_get_include}' +incdir_numpy = run_command(py, + ['-c', 'import os; os.chdir(".."); import numpy; print(numpy.get_include())'], + check : true +).stdout().strip() -incdir_f2py = '${f2py_get_include}' +incdir_f2py = run_command(py, + ['-c', 'import os; os.chdir(".."); import numpy.f2py; print(numpy.f2py.get_include())'], + check : true +).stdout().strip() + +inc_np = include_directories(incdir_numpy) +np_dep = declare_dependency(include_directories: inc_np) + +incdir_f2py = incdir_numpy / '..' / '..' / 'f2py' / 'src' +inc_f2py = include_directories(incdir_f2py) +fortranobject_c = incdir_f2py / 'fortranobject.c' inc_np = include_directories(incdir_numpy, incdir_f2py) -py3.extension_module('${modulename}', ${wrappers}, ${source_list}, incdir_f2py+'/fortranobject.c', include_directories: [inc_np, ${include_directories_list}], dependencies : [py3_dep, ${dependencies_list}], link_args : '${linker_args}', c_args : [${macros}], install : true) +py3.extension_module('${modulename}', + ${wrappers}, + ${source_list}, + fortranobject_c, + include_directories: [inc_np, ${include_directories_list}], + dependencies : [py3_dep, ${dependencies_list}], + link_args : '${linker_args}', + c_args : [${macros}], + install : true) From 323fa5b64297ecde64f23c82b287a63717bdc9ca Mon Sep 17 00:00:00 2001 From: Rohit Goswami Date: Fri, 25 Aug 2023 01:05:51 +0000 Subject: [PATCH 11/37] MAINT: Add objects --- numpy/f2py/backends/src/meson.build.src | 1 + 1 file changed, 1 insertion(+) diff --git a/numpy/f2py/backends/src/meson.build.src b/numpy/f2py/backends/src/meson.build.src index b1f87818ed7d..d64146b3c040 100644 --- a/numpy/f2py/backends/src/meson.build.src +++ b/numpy/f2py/backends/src/meson.build.src @@ -37,6 +37,7 @@ py3.extension_module('${modulename}', fortranobject_c, include_directories: [inc_np, ${include_directories_list}], dependencies : [py3_dep, ${dependencies_list}], + objects : ${objects_list}, link_args : '${linker_args}', c_args : [${macros}], install : true) From 8d4368b712e860b9d9ab1019e7e3bdb8de73bee4 Mon Sep 17 00:00:00 2001 From: Rohit Goswami Date: Fri, 25 Aug 2023 01:12:42 +0000 Subject: [PATCH 12/37] MAINT: Cleanup distutils backend --- numpy/f2py/backends/distutils_backend.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/numpy/f2py/backends/distutils_backend.py b/numpy/f2py/backends/distutils_backend.py index 83bd8558e5db..263c59223cf9 100644 --- a/numpy/f2py/backends/distutils_backend.py +++ b/numpy/f2py/backends/distutils_backend.py @@ -3,6 +3,7 @@ from numpy.distutils.core import setup, Extension from numpy.distutils.system_info import get_info from numpy.distutils.misc_util import dict_append +from numpy.exceptions import VisibleDeprecationWarning import os import sys import shutil @@ -15,7 +16,9 @@ def __init__(sef, *args, **kwargs): "distutils has been deprecated since NumPy 2.16." "Use the MesonBackend instead, or generate wrappers" "without -c and use a custom build script", - np.VisibleDeprecationWarning, stacklevel=2) + VisibleDeprecationWarning, + stacklevel=2, + ) super().__init__(*args, **kwargs) def compile(self, sources, extra_objects, build_dir): From 7bd8c34bf3192f31fec8ded88b94529c66903e97 Mon Sep 17 00:00:00 2001 From: Rohit Goswami Date: Fri, 25 Aug 2023 01:45:54 +0000 Subject: [PATCH 13/37] MAINT: Refactor to add everything back to backend Necessary for the meson.build for now. Refactors / cleanup needs better argument handling in f2py2e --- numpy/f2py/backends/backend.py | 8 +++++++- numpy/f2py/backends/distutils_backend.py | 17 +++++++++-------- numpy/f2py/f2py2e.py | 24 +++++++++++++++++++----- 3 files changed, 35 insertions(+), 14 deletions(-) diff --git a/numpy/f2py/backends/backend.py b/numpy/f2py/backends/backend.py index d86789d1915f..9fea9d243137 100644 --- a/numpy/f2py/backends/backend.py +++ b/numpy/f2py/backends/backend.py @@ -7,6 +7,9 @@ class Backend(ABC): def __init__( self, modulename, + sources, + extra_objects, + build_dir, include_dirs, library_dirs, libraries, @@ -20,6 +23,9 @@ def __init__( remove_build_dir, ): self.modulename = modulename + self.sources = sources + self.extra_objects = extra_objects + self.build_dir = build_dir self.include_dirs = include_dirs self.library_dirs = library_dirs self.libraries = libraries @@ -33,6 +39,6 @@ def __init__( self.remove_build_dir = remove_build_dir @abstractmethod - def compile(self, sources, extra_objects, build_dir) -> None: + def compile(self) -> None: """Compile the wrapper.""" pass diff --git a/numpy/f2py/backends/distutils_backend.py b/numpy/f2py/backends/distutils_backend.py index 263c59223cf9..8f79682f2d34 100644 --- a/numpy/f2py/backends/distutils_backend.py +++ b/numpy/f2py/backends/distutils_backend.py @@ -21,19 +21,20 @@ def __init__(sef, *args, **kwargs): ) super().__init__(*args, **kwargs) - def compile(self, sources, extra_objects, build_dir): + def compile(self): num_info = {} if num_info: self.include_dirs.extend(num_info.get("include_dirs", [])) ext_args = { "name": self.modulename, - "sources": sources, + "sources": self.sources, "include_dirs": self.include_dirs, "library_dirs": self.library_dirs, "libraries": self.libraries, "define_macros": self.define_macros, "undef_macros": self.undef_macros, - "extra_objects": extra_objects, + # TODO: Handle extra_objects in meson + "extra_objects": self.extra_objects, "f2py_options": self.f2py_flags, } @@ -53,9 +54,9 @@ def compile(self, sources, extra_objects, build_dir): [ "build", "--build-temp", - build_dir, + self.build_dir, "--build-base", - build_dir, + self.build_dir, "--build-platlib", ".", "--disable-optimization", @@ -69,6 +70,6 @@ def compile(self, sources, extra_objects, build_dir): setup(ext_modules=[ext]) - if self.remove_build_dir and os.path.exists(build_dir): - print(f"Removing build directory {build_dir}") - shutil.rmtree(build_dir) + if self.remove_build_dir and os.path.exists(self.build_dir): + print(f"Removing build directory {self.build_dir}") + shutil.rmtree(self.build_dir) diff --git a/numpy/f2py/f2py2e.py b/numpy/f2py/f2py2e.py index 60fc2913dacd..0cb59e279e85 100755 --- a/numpy/f2py/f2py2e.py +++ b/numpy/f2py/f2py2e.py @@ -646,11 +646,25 @@ def run_compile(): include_dirs.extend(num_info.get('include_dirs', [])) # Now use the builder - builder = build_backend(modulename, include_dirs, library_dirs, libraries, - define_macros, undef_macros, f2py_flags, - sysinfo_flags, fc_flags, flib_flags, setup_flags, remove_build_dir) - - builder.compile(sources, extra_objects, build_dir) + builder = build_backend( + modulename, + sources, + extra_objects, + build_dir, + include_dirs, + library_dirs, + libraries, + define_macros, + undef_macros, + f2py_flags, + sysinfo_flags, + fc_flags, + flib_flags, + setup_flags, + remove_build_dir, + ) + + builder.compile() def main(): if '--help-link' in sys.argv[1:]: From d7a1ea98b405a1fca4e7d9ad8e7096c35599563a Mon Sep 17 00:00:00 2001 From: Rohit Goswami Date: Fri, 25 Aug 2023 03:39:11 +0000 Subject: [PATCH 14/37] MAINT: Fix overly long line --- numpy/f2py/backends/distutils_backend.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/numpy/f2py/backends/distutils_backend.py b/numpy/f2py/backends/distutils_backend.py index 8f79682f2d34..4e5aac72f398 100644 --- a/numpy/f2py/backends/distutils_backend.py +++ b/numpy/f2py/backends/distutils_backend.py @@ -43,7 +43,8 @@ def compile(self): i = get_info(n) if not i: print( - f"No {repr(n)} resources found in system (try `f2py --help-link`)" + f"No {repr(n)} resources found" + "in system (try `f2py --help-link`)" ) dict_append(ext_args, **i) From a95e71a7a9f3d71dab756e9de55b44344443d15d Mon Sep 17 00:00:00 2001 From: Rohit Goswami Date: Fri, 25 Aug 2023 03:39:34 +0000 Subject: [PATCH 15/37] BUG: Construct wrappers for meson backend --- numpy/f2py/f2py2e.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/numpy/f2py/f2py2e.py b/numpy/f2py/f2py2e.py index 0cb59e279e85..a78a6be6123c 100755 --- a/numpy/f2py/f2py2e.py +++ b/numpy/f2py/f2py2e.py @@ -639,11 +639,11 @@ def run_compile(): else: print('Invalid use of -D:', name_value) - from numpy.distutils.system_info import get_info - - num_info = {} - if num_info: - include_dirs.extend(num_info.get('include_dirs', [])) + # Construct wrappers / signatures / things + if backend_key == 'meson': + outmess('Using meson backend\nWill pass --lower to f2py\nSee https://numpy.org/doc/stable/f2py/buildtools/meson.html') + f2py_flags.append('--lower') + run_main(f" {' '.join(f2py_flags)} -m {modulename} {' '.join(sources)}".split()) # Now use the builder builder = build_backend( From 06210325050831ee60c7c96c8a8ee866b7533385 Mon Sep 17 00:00:00 2001 From: Rohit Goswami Date: Fri, 25 Aug 2023 03:40:14 +0000 Subject: [PATCH 16/37] MAINT: Rework, simplify template massively --- numpy/f2py/backends/src/meson.build.src | 29 +++++++++---------------- 1 file changed, 10 insertions(+), 19 deletions(-) diff --git a/numpy/f2py/backends/src/meson.build.src b/numpy/f2py/backends/src/meson.build.src index d64146b3c040..84b8f169dbb6 100644 --- a/numpy/f2py/backends/src/meson.build.src +++ b/numpy/f2py/backends/src/meson.build.src @@ -1,12 +1,8 @@ -project('${modulename}', 'c', - version : '0.1', - meson_version: '>= 1.1.0', - default_options : ['warning_level=2']) - -add_languages('fortran') - -add_project_arguments(${c_global_args}, language: 'c') -add_project_arguments(${fortran_global_args}, language: 'fortran') +project('${modulename}', + ['c', 'fortran'], + version : '0.1', + meson_version: '>= 1.1.0', + default_options : ['warning_level=2']) py_mod = import('python') py = py_mod.find_installation(pure: false) @@ -31,13 +27,8 @@ fortranobject_c = incdir_f2py / 'fortranobject.c' inc_np = include_directories(incdir_numpy, incdir_f2py) -py3.extension_module('${modulename}', - ${wrappers}, - ${source_list}, - fortranobject_c, - include_directories: [inc_np, ${include_directories_list}], - dependencies : [py3_dep, ${dependencies_list}], - objects : ${objects_list}, - link_args : '${linker_args}', - c_args : [${macros}], - install : true) +py.extension_module('${modulename}', + [${source_list}, fortranobject_c], + include_directories: [inc_np], + dependencies : [py_dep], + install : true) From 677e42a50efe28cf6b26f4e2b6127ddb4deb3df7 Mon Sep 17 00:00:00 2001 From: Rohit Goswami Date: Fri, 25 Aug 2023 03:40:26 +0000 Subject: [PATCH 17/37] ENH: Truncate meson.build to skeleton only --- numpy/f2py/backends/meson_backend.py | 293 ++++++--------------------- 1 file changed, 61 insertions(+), 232 deletions(-) diff --git a/numpy/f2py/backends/meson_backend.py b/numpy/f2py/backends/meson_backend.py index df0d1b4fdb8b..49820f036e8f 100644 --- a/numpy/f2py/backends/meson_backend.py +++ b/numpy/f2py/backends/meson_backend.py @@ -1,6 +1,5 @@ from __future__ import annotations -import os import errno import shutil import subprocess @@ -9,228 +8,71 @@ from .backend import Backend from string import Template +import warnings + class MesonTemplate: """Template meson build file generation class.""" def __init__( self, - module_name: str, - numpy_install_path, - numpy_get_include: Path, - f2py_get_include: Path, - wrappers: list[Path], - fortran_sources: list[Path], - dependencies: list[str], - include_path: list[Path], - optimization_flags: list[str], - architecture_flags: list[str], - f77_flags: list[str], - f90_flags: list[str], - linker_libpath: list[Path], - linker_libname: list[str], - define_macros: list[tuple[str, str]], - undef_macros: list[str], + modulename: str, + sources: list[Path], + object_files: list[Path], + linker_args: list[str], + c_args: list[str], ): - self.module_name = module_name - self.numpy_install_path = numpy_install_path + self.modulename = modulename self.build_template_path = ( - numpy_install_path / "f2py" / "backends" / "src" / "meson.build.src" + Path(__file__).parent.absolute() / "src" / "meson.build.src" ) - self.sources = fortran_sources - self.numpy_get_include = numpy_get_include - self.f2py_get_include = f2py_get_include - self.wrappers = wrappers - self.dependencies = dependencies - self.include_directories = include_path + self.sources = sources self.substitutions = {} - self.optimization_flags = optimization_flags - self.architecture_flags = architecture_flags - self.fortran_flags = f77_flags + f90_flags - self.linker_libpath = linker_libpath - self.linker_libname = linker_libname - self.define_macros = define_macros - self.undef_macros = undef_macros + self.objects = object_files self.pipeline = [ self.initialize_template, - self.global_flags_substitution, self.sources_substitution, - self.dependencies_substitution, - self.include_directories_subtitution, - self.linker_substitution, - self.macros_substitution, ] - @property def meson_build_template(self) -> str: if not self.build_template_path.is_file(): raise FileNotFoundError( errno.ENOENT, - f"Meson build template {self.build_template_path.absolute()} does not exist.", + "Meson build template" + f" {self.build_template_path.absolute()}" + " does not exist.", ) return self.build_template_path.read_text() def initialize_template(self) -> None: - """Initialize with module name and external NumPy and F2PY C libraries.""" - self.substitutions["modulename"] = self.module_name - self.substitutions["numpy_get_include"] = self.numpy_get_include.absolute() - self.substitutions["f2py_get_include"] = self.f2py_get_include.absolute() + """Initialize with module name.""" + self.substitutions["modulename"] = self.modulename def sources_substitution(self) -> None: - self.substitutions["source_list"] = ",".join( - ["'" + str(source.absolute()) + "'" for source in self.sources] - ) - self.substitutions["wrappers"] = ",".join( - ["'" + str(wrapper.absolute()) + "'" for wrapper in self.wrappers] - ) - - def dependencies_substitution(self) -> None: - self.substitutions["dependencies_list"] = ", ".join( - [f"dependency('{dependecy}')" for dependecy in self.dependencies] - ) - - def include_directories_subtitution(self) -> None: - self.substitutions["include_directories_list"] = ", ".join( - [ - f"include_directories('{include_directory}')" - for include_directory in self.include_directories - ] - ) - - def global_flags_substitution(self) -> None: - fortran_compiler_flags = ( - self.fortran_flags + self.optimization_flags + self.architecture_flags + self.substitutions["source_list"] = ", ".join( + [f"'{source}'" for source in self.sources] ) - c_compiler_flags = self.optimization_flags + self.architecture_flags - self.substitutions["fortran_global_args"] = fortran_compiler_flags - self.substitutions["c_global_args"] = c_compiler_flags - - def macros_substitution(self) -> None: - self.substitutions["macros"] = "" - if self.define_macros: - self.substitutions["macros"] = ",".join( - f"'-D{macro[0]}={macro[1]}'" if macro[1] else f"-D{macro[0]}" - for macro in self.define_macros - ) - if self.undef_macros: - self.substitutions["macros"] += "," + ",".join( - f"'-U{macro}'" for macro in self.undef_macros - ) - def linker_substitution(self) -> None: - self.substitutions["linker_args"] = "" - if self.linker_libpath: - linker_libpath_subs = ",".join( - f"-L{libpath}" for libpath in self.linker_libpath - ) - self.substitutions["linker_args"] += linker_libpath_subs - if self.linker_libname: - linker_libname_subs = ",".join( - f"-l{libname}" for libname in self.linker_libname - ) - self.substitutions["linker_args"] += f",{linker_libname_subs}" - - def generate_meson_build(self) -> str: + def generate_meson_build(self): for node in self.pipeline: node() - template = Template(self.meson_build_template) + template = Template(self.meson_build_template()) return template.substitute(self.substitutions) class MesonBackend(Backend): - def __init__( - self, - module_name: str = "untitled", - fortran_compiler: str = None, - c_compiler: str = None, - f77exec: Path = None, - f90exec: Path = None, - f77_flags: list[str] = None, - f90_flags: list[str] = None, - include_paths: list[Path] = None, - include_dirs: list[Path] = None, - external_resources: list[str] = None, - linker_libpath: list[Path] = None, - linker_libname: list[str] = None, - define_macros: list[tuple[str, str]] = None, - undef_macros: list[str] = None, - debug: bool = False, - opt_flags: list[str] = None, - arch_flags: list[str] = None, - no_opt: bool = False, - no_arch: bool = False, - ) -> None: - self.meson_build_dir = "builddir" - if f77_flags is None: - f77_flags = [] - if include_paths is None: - include_paths = [] - if include_dirs is None: - include_dirs = [] - if external_resources is None: - external_resources = [] - if linker_libpath is None: - linker_libpath = [] - if linker_libname is None: - linker_libname = [] - if define_macros is None: - define_macros = [] - if undef_macros is None: - undef_macros = [] - if f77_flags is None: - f77_flags = [] - if f90_flags is None: - f90_flags = [] - if opt_flags is None: - opt_flags = [] - if arch_flags is None: - arch_flags = [] - super().__init__( - module_name, - fortran_compiler, - c_compiler, - f77exec, - f90exec, - f77_flags, - f90_flags, - include_paths, - include_dirs, - external_resources, - linker_libpath, - linker_libname, - define_macros, - undef_macros, - debug, - opt_flags, - arch_flags, - no_opt, - no_arch, + def __init__(self, *args, **kwargs): + warnings.warn( + "The MesonBackend is experimental and may change in the future." + "--build-dir should be used to customize the generated skeleton.", + stacklevel=2, ) - - self.wrappers: list[Path] = [] - self.fortran_sources: list[Path] = [] - self.template = Template(self.fortran_sources) - - def _get_optimization_level(self): - if self.no_arch and not self.no_opt: - return 2 - elif self.no_opt: - return 0 - return 3 - - def _set_environment_variables(self) -> None: - if self.fortran_compiler: - os.putenv("FC", self.fortran_compiler) - elif self.f77exec: - os.putenv("FC", self.f77exec) - elif self.f90exec: - os.putenv("FC", self.f90exec) - if self.c_compiler: - os.putenv("CC", self.c_compiler) + super().__init__(*args, **kwargs) + self.meson_build_dir = "bbdir" def _move_exec_to_root(self, build_dir: Path): - walk_dir = build_dir / self.meson_build_dir - path_objects = walk_dir.glob(f"{self.module_name}*.so") + walk_dir = Path(build_dir) / self.meson_build_dir + path_objects = walk_dir.glob(f"{self.modulename}*.so") for path_object in path_objects: shutil.move(path_object, Path.cwd()) @@ -239,49 +81,24 @@ def _get_build_command(self): "meson", "setup", self.meson_build_dir, - "-Ddebug=true" if self.debug else "-Ddebug=false", - f"-Doptimization={str(self._get_optimization_level())}", ] - def load_wrapper(self, wrappers: list[Path]) -> None: - self.wrappers = wrappers - - def load_sources(self, fortran_sources: list[Path]) -> None: - for fortran_source in fortran_sources: - fortran_source = Path(fortran_source) - if not fortran_source.is_file(): - raise FileNotFoundError( - errno.ENOENT, f"{fortran_source.absolute()} does not exist." - ) - self.fortran_sources.append(fortran_source) - def write_meson_build(self, build_dir: Path) -> None: """Writes the meson build file at specified location""" meson_template = MesonTemplate( - self.module_name, - super().numpy_install_path(), - self.numpy_get_include(), - self.f2py_get_include(), - self.wrappers, - self.fortran_sources, - self.external_resources, - self.include_paths + self.include_dirs, - optimization_flags=self.opt_flags, - architecture_flags=self.arch_flags, - f77_flags=self.f77_flags, - f90_flags=self.f90_flags, - linker_libpath=self.linker_libpath, - linker_libname=self.linker_libname, - define_macros=self.define_macros, - undef_macros=self.undef_macros, + self.modulename, + self.sources, + self.extra_objects, + self.flib_flags, + self.fc_flags, ) src = meson_template.generate_meson_build() - meson_build_file = build_dir / "meson.build" + Path(build_dir).mkdir(parents=True, exist_ok=True) + meson_build_file = Path(build_dir) / "meson.build" meson_build_file.write_text(src) return meson_build_file def run_meson(self, build_dir: Path): - self._set_environment_variables() completed_process = subprocess.run(self._get_build_command(), cwd=build_dir) if completed_process.returncode != 0: raise subprocess.CalledProcessError( @@ -295,16 +112,28 @@ def run_meson(self, build_dir: Path): completed_process.returncode, completed_process.args ) - def compile( - self, - f77_sources: list[Path], - f90_sources: list[Path], - object_files: list[Path], - wrappers: list[Path], - build_dir: Path, - ) -> None: - self.load_wrapper(wrappers) - self.load_sources(f77_sources + f90_sources + object_files) - self.write_meson_build(build_dir) - self.run_meson(build_dir) - self._move_exec_to_root(build_dir) + def compile(self) -> None: + self.sources = _prepare_sources(self.modulename, self.sources, self.build_dir) + self.write_meson_build(self.build_dir) + self.run_meson(self.build_dir) + self._move_exec_to_root(self.build_dir) + + +def _prepare_sources(mname, sources, bdir): + extended_sources = sources.copy() + Path(bdir).mkdir(parents=True, exist_ok=True) + # Copy sources + for source in sources: + shutil.copy(source, bdir) + generated_sources = [ + Path(f"{mname}module.c"), + Path(f"{mname}-f2pywrappers2.f90"), + Path(f"{mname}-f2pywrappers.f"), + ] + bdir = Path(bdir) + for generated_source in generated_sources: + if generated_source.exists(): + shutil.copy(generated_source, bdir / generated_source.name) + extended_sources.append(bdir / generated_source.name) + generated_source.unlink() + return extended_sources From 2517afeace4aea2070d858e9184f0d84eb644a50 Mon Sep 17 00:00:00 2001 From: Rohit Goswami Date: Fri, 25 Aug 2023 03:58:10 +0000 Subject: [PATCH 18/37] MAINT: Minor backend housekeeping, name changes --- numpy/f2py/{backends => _backends}/__init__.py | 8 +++----- numpy/f2py/{backends/backend.py => _backends/_backend.py} | 0 .../distutils_backend.py => _backends/_distutils.py} | 2 +- .../{backends/meson_backend.py => _backends/_meson.py} | 4 ++-- numpy/f2py/{backends => _backends}/src/meson.build.src | 0 numpy/f2py/f2py2e.py | 4 ++-- 6 files changed, 8 insertions(+), 10 deletions(-) rename numpy/f2py/{backends => _backends}/__init__.py (57%) rename numpy/f2py/{backends/backend.py => _backends/_backend.py} (100%) rename numpy/f2py/{backends/distutils_backend.py => _backends/_distutils.py} (98%) rename numpy/f2py/{backends/meson_backend.py => _backends/_meson.py} (97%) rename numpy/f2py/{backends => _backends}/src/meson.build.src (100%) diff --git a/numpy/f2py/backends/__init__.py b/numpy/f2py/_backends/__init__.py similarity index 57% rename from numpy/f2py/backends/__init__.py rename to numpy/f2py/_backends/__init__.py index cd3a6a93f002..e91393c14be3 100644 --- a/numpy/f2py/backends/__init__.py +++ b/numpy/f2py/_backends/__init__.py @@ -1,11 +1,9 @@ -def get_backend(name): +def f2py_build_generator(name): if name == "meson": - from .meson_backend import MesonBackend - + from ._meson import MesonBackend return MesonBackend elif name == "distutils": - from .distutils_backend import DistutilsBackend - + from ._distutils import DistutilsBackend return DistutilsBackend else: raise ValueError(f"Unknown backend: {name}") diff --git a/numpy/f2py/backends/backend.py b/numpy/f2py/_backends/_backend.py similarity index 100% rename from numpy/f2py/backends/backend.py rename to numpy/f2py/_backends/_backend.py diff --git a/numpy/f2py/backends/distutils_backend.py b/numpy/f2py/_backends/_distutils.py similarity index 98% rename from numpy/f2py/backends/distutils_backend.py rename to numpy/f2py/_backends/_distutils.py index 4e5aac72f398..efc7a98b7692 100644 --- a/numpy/f2py/backends/distutils_backend.py +++ b/numpy/f2py/_backends/_distutils.py @@ -1,4 +1,4 @@ -from numpy.f2py.backends.backend import Backend +from ._backend import Backend from numpy.distutils.core import setup, Extension from numpy.distutils.system_info import get_info diff --git a/numpy/f2py/backends/meson_backend.py b/numpy/f2py/_backends/_meson.py similarity index 97% rename from numpy/f2py/backends/meson_backend.py rename to numpy/f2py/_backends/_meson.py index 49820f036e8f..86e0a1a3943c 100644 --- a/numpy/f2py/backends/meson_backend.py +++ b/numpy/f2py/_backends/_meson.py @@ -5,7 +5,7 @@ import subprocess from pathlib import Path -from .backend import Backend +from ._backend import Backend from string import Template import warnings @@ -134,6 +134,6 @@ def _prepare_sources(mname, sources, bdir): for generated_source in generated_sources: if generated_source.exists(): shutil.copy(generated_source, bdir / generated_source.name) - extended_sources.append(bdir / generated_source.name) + extended_sources.append(Path(bdir / generated_source.name).absolute()) generated_source.unlink() return extended_sources diff --git a/numpy/f2py/backends/src/meson.build.src b/numpy/f2py/_backends/src/meson.build.src similarity index 100% rename from numpy/f2py/backends/src/meson.build.src rename to numpy/f2py/_backends/src/meson.build.src diff --git a/numpy/f2py/f2py2e.py b/numpy/f2py/f2py2e.py index a78a6be6123c..0a6d164b4890 100755 --- a/numpy/f2py/f2py2e.py +++ b/numpy/f2py/f2py2e.py @@ -28,7 +28,7 @@ from . import f90mod_rules from . import __version__ from . import capi_maps -from . import backends +from . import _backends f2py_version = __version__.version numpy_version = __version__.version @@ -598,7 +598,7 @@ def run_compile(): sys.argv.pop(backend_index) else: backend_key = 'distutils' - build_backend = backends.get_backend(backend_key) + build_backend = _backends.f2py_build_generator(backend_key) modulename = 'untitled' sources = sys.argv[1:] From 1dc27333ffdf1e39c5d8badb90feadc9fc6562ce Mon Sep 17 00:00:00 2001 From: Rohit Goswami Date: Fri, 25 Aug 2023 04:57:33 +0000 Subject: [PATCH 19/37] MAINT: Less absolute paths, update setup.py [f2py] --- numpy/f2py/f2py2e.py | 4 ++-- numpy/f2py/setup.py | 5 ++++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/numpy/f2py/f2py2e.py b/numpy/f2py/f2py2e.py index 0a6d164b4890..6bd41327a214 100755 --- a/numpy/f2py/f2py2e.py +++ b/numpy/f2py/f2py2e.py @@ -28,7 +28,7 @@ from . import f90mod_rules from . import __version__ from . import capi_maps -from . import _backends +from numpy.f2py._backends import f2py_build_generator f2py_version = __version__.version numpy_version = __version__.version @@ -598,7 +598,7 @@ def run_compile(): sys.argv.pop(backend_index) else: backend_key = 'distutils' - build_backend = _backends.f2py_build_generator(backend_key) + build_backend = f2py_build_generator(backend_key) modulename = 'untitled' sources = sys.argv[1:] diff --git a/numpy/f2py/setup.py b/numpy/f2py/setup.py index 499609f96600..075d4683699e 100644 --- a/numpy/f2py/setup.py +++ b/numpy/f2py/setup.py @@ -26,10 +26,13 @@ def configuration(parent_package='', top_path=None): config = Configuration('f2py', parent_package, top_path) config.add_subpackage('tests') + config.add_subpackage('_backends') config.add_data_dir('tests/src') config.add_data_files( 'src/fortranobject.c', - 'src/fortranobject.h') + 'src/fortranobject.h', + 'backends/src/meson.build.src', + ) config.add_data_files('*.pyi') return config From 42a15acfd7ee398841c2043e8e3c5e3ab9123338 Mon Sep 17 00:00:00 2001 From: Rohit Goswami Date: Fri, 25 Aug 2023 14:33:44 +0000 Subject: [PATCH 20/37] MAINT: Move f2py module name functionality Previously part of np.distutils --- numpy/f2py/auxfuncs.py | 20 +++++++++++++++++++- numpy/f2py/f2py2e.py | 3 +-- 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/numpy/f2py/auxfuncs.py b/numpy/f2py/auxfuncs.py index c0864b5bc613..535e324286bd 100644 --- a/numpy/f2py/auxfuncs.py +++ b/numpy/f2py/auxfuncs.py @@ -16,6 +16,7 @@ """ import pprint import sys +import re import types from functools import reduce from copy import deepcopy @@ -43,7 +44,7 @@ 'ismodule', 'ismoduleroutine', 'isoptional', 'isprivate', 'isrequired', 'isroutine', 'isscalar', 'issigned_long_longarray', 'isstring', 'isstringarray', 'isstring_or_stringarray', 'isstringfunction', - 'issubroutine', + 'issubroutine', 'get_f2py_modulename', 'issubroutine_wrap', 'isthreadsafe', 'isunsigned', 'isunsigned_char', 'isunsigned_chararray', 'isunsigned_long_long', 'isunsigned_long_longarray', 'isunsigned_short', @@ -912,3 +913,20 @@ def deep_merge(dict1, dict2): else: merged_dict[key] = value return merged_dict + +_f2py_module_name_match = re.compile(r'\s*python\s*module\s*(?P[\w_]+)', + re.I).match +_f2py_user_module_name_match = re.compile(r'\s*python\s*module\s*(?P[\w_]*?' + r'__user__[\w_]*)', re.I).match + +def get_f2py_modulename(source): + name = None + with open(source) as f: + for line in f: + m = _f2py_module_name_match(line) + if m: + if _f2py_user_module_name_match(line): # skip *__user__* names + continue + name = m.group('name') + break + return name diff --git a/numpy/f2py/f2py2e.py b/numpy/f2py/f2py2e.py index 6bd41327a214..ef856fd9ee99 100755 --- a/numpy/f2py/f2py2e.py +++ b/numpy/f2py/f2py2e.py @@ -616,11 +616,10 @@ def run_compile(): del sys.argv[i + 1], sys.argv[i] sources = sys.argv[1:] else: - from numpy.distutils.command.build_src import get_f2py_modulename pyf_files, sources = filter_files('', '[.]pyf([.]src|)', sources) sources = pyf_files + sources for f in pyf_files: - modulename = get_f2py_modulename(f) + modulename = auxfuncs.get_f2py_modulename(f) if modulename: break From 6776dba6e1bd8f0769f629b25d8c37f8f5938f43 Mon Sep 17 00:00:00 2001 From: Rohit Goswami Date: Fri, 25 Aug 2023 15:16:12 +0000 Subject: [PATCH 21/37] ENH: Handle .pyf files --- numpy/f2py/_backends/_meson.py | 3 +++ numpy/f2py/f2py2e.py | 9 ++++++--- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/numpy/f2py/_backends/_meson.py b/numpy/f2py/_backends/_meson.py index 86e0a1a3943c..2dda014d001e 100644 --- a/numpy/f2py/_backends/_meson.py +++ b/numpy/f2py/_backends/_meson.py @@ -136,4 +136,7 @@ def _prepare_sources(mname, sources, bdir): shutil.copy(generated_source, bdir / generated_source.name) extended_sources.append(Path(bdir / generated_source.name).absolute()) generated_source.unlink() + extended_sources = [ + source for source in extended_sources if not Path(source).suffix == ".pyf" + ] return extended_sources diff --git a/numpy/f2py/f2py2e.py b/numpy/f2py/f2py2e.py index ef856fd9ee99..da509bfb515e 100755 --- a/numpy/f2py/f2py2e.py +++ b/numpy/f2py/f2py2e.py @@ -616,8 +616,8 @@ def run_compile(): del sys.argv[i + 1], sys.argv[i] sources = sys.argv[1:] else: - pyf_files, sources = filter_files('', '[.]pyf([.]src|)', sources) - sources = pyf_files + sources + pyf_files, _sources = filter_files('', '[.]pyf([.]src|)', sources) + sources = pyf_files + _sources for f in pyf_files: modulename = auxfuncs.get_f2py_modulename(f) if modulename: @@ -642,7 +642,10 @@ def run_compile(): if backend_key == 'meson': outmess('Using meson backend\nWill pass --lower to f2py\nSee https://numpy.org/doc/stable/f2py/buildtools/meson.html') f2py_flags.append('--lower') - run_main(f" {' '.join(f2py_flags)} -m {modulename} {' '.join(sources)}".split()) + if pyf_files: + run_main(f" {' '.join(f2py_flags)} {' '.join(pyf_files)}".split()) + else: + run_main(f" {' '.join(f2py_flags)} -m {modulename} {' '.join(sources)}".split()) # Now use the builder builder = build_backend( From 0357ab8ed8e09c06ab1c10627762bf34a0265f23 Mon Sep 17 00:00:00 2001 From: Rohit Goswami Date: Fri, 25 Aug 2023 15:51:18 +0000 Subject: [PATCH 22/37] TST: Fix typo in isoFortranEnvMap.f90 --- numpy/f2py/tests/src/f2cmap/isoFortranEnvMap.f90 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/numpy/f2py/tests/src/f2cmap/isoFortranEnvMap.f90 b/numpy/f2py/tests/src/f2cmap/isoFortranEnvMap.f90 index 3f0e12c76833..1e1dc1d4054b 100644 --- a/numpy/f2py/tests/src/f2cmap/isoFortranEnvMap.f90 +++ b/numpy/f2py/tests/src/f2cmap/isoFortranEnvMap.f90 @@ -4,6 +4,6 @@ subroutine func1(n, x, res) integer(int64), intent(in) :: n real(real64), intent(in) :: x(n) real(real64), intent(out) :: res -Cf2py intent(hide) :: n +!f2py intent(hide) :: n res = sum(x) end From 5faec2f6d4b8973525e75f137b3e6aec6c0e3a20 Mon Sep 17 00:00:00 2001 From: Rohit Goswami Date: Fri, 25 Aug 2023 15:52:20 +0000 Subject: [PATCH 23/37] MAINT: Typo in f2py2e support for pyf files --- numpy/f2py/f2py2e.py | 1 + 1 file changed, 1 insertion(+) diff --git a/numpy/f2py/f2py2e.py b/numpy/f2py/f2py2e.py index da509bfb515e..6b63a6c6b0b0 100755 --- a/numpy/f2py/f2py2e.py +++ b/numpy/f2py/f2py2e.py @@ -610,6 +610,7 @@ def run_compile(): del sys.argv[i + 1], sys.argv[i] sources = sys.argv[1:] + pyf_files = [] if '-m' in sys.argv: i = sys.argv.index('-m') modulename = sys.argv[i + 1] From 5b794876c933ead944544c7450888ed9635f03cc Mon Sep 17 00:00:00 2001 From: Rohit Goswami Date: Fri, 25 Aug 2023 16:13:01 +0000 Subject: [PATCH 24/37] DOC: Add release note for --backend --- doc/release/upcoming_changes/24532.new_feature.rst | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 doc/release/upcoming_changes/24532.new_feature.rst diff --git a/doc/release/upcoming_changes/24532.new_feature.rst b/doc/release/upcoming_changes/24532.new_feature.rst new file mode 100644 index 000000000000..6c89d9dd7574 --- /dev/null +++ b/doc/release/upcoming_changes/24532.new_feature.rst @@ -0,0 +1,5 @@ +``meson`` backend for ``f2py`` +------------------------------ +``f2py`` now accepts the ``--backend meson`` option. This is the default option +for Python ``3.12`` on-wards. Older versions will still default to ``--backend +distutils``. From faadb6d61013002885125e0b52f86800617890b3 Mon Sep 17 00:00:00 2001 From: Rohit Goswami Date: Fri, 25 Aug 2023 16:18:19 +0000 Subject: [PATCH 25/37] MAINT: Conditional switch for Python 3.12 [f2py] --- numpy/f2py/f2py2e.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/numpy/f2py/f2py2e.py b/numpy/f2py/f2py2e.py index 6b63a6c6b0b0..43fbcc66f6c6 100755 --- a/numpy/f2py/f2py2e.py +++ b/numpy/f2py/f2py2e.py @@ -598,6 +598,10 @@ def run_compile(): sys.argv.pop(backend_index) else: backend_key = 'distutils' + + if sys.version_info >= (3, 12) and backend_key == 'distutils': + outmess('Cannot use distutils backend with Python 3.12, using meson backend instead.') + backend_key = 'meson' build_backend = f2py_build_generator(backend_key) modulename = 'untitled' From 82a4f8fd347f8e85f1aab363a67494f9fde2006f Mon Sep 17 00:00:00 2001 From: Rohit Goswami Date: Fri, 25 Aug 2023 21:47:47 +0000 Subject: [PATCH 26/37] MAINT: No absolute paths in backend [f2py-meson] The files are copied over anyway, this makes it easier to extend the generated skeleton --- numpy/f2py/_backends/_meson.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/numpy/f2py/_backends/_meson.py b/numpy/f2py/_backends/_meson.py index 2dda014d001e..90bdce86f0ed 100644 --- a/numpy/f2py/_backends/_meson.py +++ b/numpy/f2py/_backends/_meson.py @@ -134,9 +134,11 @@ def _prepare_sources(mname, sources, bdir): for generated_source in generated_sources: if generated_source.exists(): shutil.copy(generated_source, bdir / generated_source.name) - extended_sources.append(Path(bdir / generated_source.name).absolute()) + extended_sources.append(generated_source.name) generated_source.unlink() extended_sources = [ - source for source in extended_sources if not Path(source).suffix == ".pyf" + Path(source).name + for source in extended_sources + if not Path(source).suffix == ".pyf" ] return extended_sources From 65854588d9dc3e0ef8e274c246bcee0ca4e30fd0 Mon Sep 17 00:00:00 2001 From: Rohit Goswami Date: Fri, 25 Aug 2023 21:53:04 +0000 Subject: [PATCH 27/37] MAINT: Prettier generated meson.build files [f2py] --- numpy/f2py/_backends/_meson.py | 4 ++-- numpy/f2py/_backends/src/meson.build.src | 5 ++++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/numpy/f2py/_backends/_meson.py b/numpy/f2py/_backends/_meson.py index 90bdce86f0ed..91fcabc0451b 100644 --- a/numpy/f2py/_backends/_meson.py +++ b/numpy/f2py/_backends/_meson.py @@ -49,8 +49,8 @@ def initialize_template(self) -> None: self.substitutions["modulename"] = self.modulename def sources_substitution(self) -> None: - self.substitutions["source_list"] = ", ".join( - [f"'{source}'" for source in self.sources] + self.substitutions["source_list"] = ",\n".join( + [f" '{source}'" for source in self.sources] ) def generate_meson_build(self): diff --git a/numpy/f2py/_backends/src/meson.build.src b/numpy/f2py/_backends/src/meson.build.src index 84b8f169dbb6..c3149164c2e7 100644 --- a/numpy/f2py/_backends/src/meson.build.src +++ b/numpy/f2py/_backends/src/meson.build.src @@ -28,7 +28,10 @@ fortranobject_c = incdir_f2py / 'fortranobject.c' inc_np = include_directories(incdir_numpy, incdir_f2py) py.extension_module('${modulename}', - [${source_list}, fortranobject_c], + [ +${source_list}, + fortranobject_c + ], include_directories: [inc_np], dependencies : [py_dep], install : true) From f85581e90f264026d47b85522e02ef000a992ae5 Mon Sep 17 00:00:00 2001 From: Rohit Goswami Date: Fri, 25 Aug 2023 22:20:50 +0000 Subject: [PATCH 28/37] ENH: Add meson's dependency(blah) to f2py --- numpy/f2py/_backends/_backend.py | 2 ++ numpy/f2py/_backends/_meson.py | 16 ++++++++++++++-- numpy/f2py/_backends/src/meson.build.src | 5 ++++- numpy/f2py/f2py2e.py | 15 ++++++++++++++- 4 files changed, 34 insertions(+), 4 deletions(-) diff --git a/numpy/f2py/_backends/_backend.py b/numpy/f2py/_backends/_backend.py index 9fea9d243137..a7d43d2587b2 100644 --- a/numpy/f2py/_backends/_backend.py +++ b/numpy/f2py/_backends/_backend.py @@ -21,6 +21,7 @@ def __init__( flib_flags, setup_flags, remove_build_dir, + extra_dat, ): self.modulename = modulename self.sources = sources @@ -37,6 +38,7 @@ def __init__( self.flib_flags = flib_flags self.setup_flags = setup_flags self.remove_build_dir = remove_build_dir + self.extra_dat = extra_dat @abstractmethod def compile(self) -> None: diff --git a/numpy/f2py/_backends/_meson.py b/numpy/f2py/_backends/_meson.py index 91fcabc0451b..eef4622c2353 100644 --- a/numpy/f2py/_backends/_meson.py +++ b/numpy/f2py/_backends/_meson.py @@ -18,6 +18,7 @@ def __init__( self, modulename: str, sources: list[Path], + deps: list[str], object_files: list[Path], linker_args: list[str], c_args: list[str], @@ -27,11 +28,13 @@ def __init__( Path(__file__).parent.absolute() / "src" / "meson.build.src" ) self.sources = sources + self.deps = deps self.substitutions = {} self.objects = object_files self.pipeline = [ self.initialize_template, self.sources_substitution, + self.deps_substitution, ] def meson_build_template(self) -> str: @@ -49,8 +52,15 @@ def initialize_template(self) -> None: self.substitutions["modulename"] = self.modulename def sources_substitution(self) -> None: - self.substitutions["source_list"] = ",\n".join( - [f" '{source}'" for source in self.sources] + indent = " " * 21 + self.substitutions["source_list"] = f",\n{indent}".join( + [f"'{source}'" for source in self.sources] + ) + + def deps_substitution(self) -> None: + indent = " " * 21 + self.substitutions["dep_list"] = f",\n{indent}".join( + [f"dependency('{dep}')" for dep in self.deps] ) def generate_meson_build(self): @@ -68,6 +78,7 @@ def __init__(self, *args, **kwargs): stacklevel=2, ) super().__init__(*args, **kwargs) + self.dependencies = self.extra_dat.get("dependencies", []) self.meson_build_dir = "bbdir" def _move_exec_to_root(self, build_dir: Path): @@ -88,6 +99,7 @@ def write_meson_build(self, build_dir: Path) -> None: meson_template = MesonTemplate( self.modulename, self.sources, + self.dependencies, self.extra_objects, self.flib_flags, self.fc_flags, diff --git a/numpy/f2py/_backends/src/meson.build.src b/numpy/f2py/_backends/src/meson.build.src index c3149164c2e7..29d711f7d37f 100644 --- a/numpy/f2py/_backends/src/meson.build.src +++ b/numpy/f2py/_backends/src/meson.build.src @@ -33,5 +33,8 @@ ${source_list}, fortranobject_c ], include_directories: [inc_np], - dependencies : [py_dep], + dependencies : [ + py_dep, +${dep_list} + ], install : true) diff --git a/numpy/f2py/f2py2e.py b/numpy/f2py/f2py2e.py index 43fbcc66f6c6..d31c87f10fbd 100755 --- a/numpy/f2py/f2py2e.py +++ b/numpy/f2py/f2py2e.py @@ -19,6 +19,7 @@ import pprint import re from pathlib import Path +from itertools import dropwhile from . import crackfortran from . import rules @@ -549,7 +550,18 @@ def run_compile(): if f2py_flags2 and f2py_flags2[-1] != ':': f2py_flags2.append(':') f2py_flags.extend(f2py_flags2) - + # Find the start and end indices of the "deps:" section + start_idx = next((i for i, a in enumerate(sys.argv) if a == "deps:"), None) + end_idx = next((i for i, a in enumerate(sys.argv) if a == ":" and i > start_idx), None) if start_idx is not None else None + # Extract dependencies and remove them from sys.argv + if start_idx is not None and end_idx is not None: + dependencies = sys.argv[start_idx + 1:end_idx] + del sys.argv[start_idx:end_idx + 1] + elif start_idx is not None: + dependencies = list(dropwhile(lambda x: x != ":", sys.argv[start_idx + 1:])) + del sys.argv[start_idx:] + else: + dependencies = [] sys.argv = [_m for _m in sys.argv if _m not in f2py_flags2] _reg3 = re.compile( r'--((f(90)?compiler(-exec|)|compiler)=|help-compiler)') @@ -669,6 +681,7 @@ def run_compile(): flib_flags, setup_flags, remove_build_dir, + {"dependencies": dependencies}, ) builder.compile() From 16f22eea8628a0be58ead8d211884de83b775f8a Mon Sep 17 00:00:00 2001 From: Rohit Goswami Date: Fri, 25 Aug 2023 22:49:56 +0000 Subject: [PATCH 29/37] DOC: Document the new flag --- numpy/f2py/f2py2e.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/numpy/f2py/f2py2e.py b/numpy/f2py/f2py2e.py index d31c87f10fbd..3a6a9c429726 100755 --- a/numpy/f2py/f2py2e.py +++ b/numpy/f2py/f2py2e.py @@ -128,7 +128,7 @@ -v Print f2py version ID and exit. -numpy.distutils options (only effective with -c): +build backend options (only effective with -c): --fcompiler= Specify Fortran compiler type by vendor --compiler= Specify C compiler type (as defined by distutils) @@ -143,6 +143,19 @@ --noopt Compile without optimization --noarch Compile without arch-dependent optimization --debug Compile with debugging information + deps: ... : + Specify additional dependencies that the module relies on. + List dependencies separated by spaces, and use `:' to + signify the end of the dependency list. Dependencies are + stored in a list for further processing. + + Example: deps: lapack scalapack : + This will identify "lapack" and "scalapack" as dependencies + and remove them from argv, leaving a dependencies list + containing ["lapack", "scalapack"]. + + Note: The trailing `:' is required to signify the end of + the dependency list. This is only relevant for the meson backend. Extra options (only effective with -c): From c0c6bf18874e54b1b0f4f25e02cb7b386e5b422e Mon Sep 17 00:00:00 2001 From: Rohit Goswami Date: Sat, 26 Aug 2023 10:26:38 +0000 Subject: [PATCH 30/37] MAINT: Simplify and rename backend template [f2py] Co-authored-by: rgommers --- numpy/f2py/_backends/_meson.py | 2 +- .../_backends/{src/meson.build.src => meson.build.template} | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename numpy/f2py/_backends/{src/meson.build.src => meson.build.template} (100%) diff --git a/numpy/f2py/_backends/_meson.py b/numpy/f2py/_backends/_meson.py index eef4622c2353..48130edbe464 100644 --- a/numpy/f2py/_backends/_meson.py +++ b/numpy/f2py/_backends/_meson.py @@ -25,7 +25,7 @@ def __init__( ): self.modulename = modulename self.build_template_path = ( - Path(__file__).parent.absolute() / "src" / "meson.build.src" + Path(__file__).parent.absolute() / "meson.build.template" ) self.sources = sources self.deps = deps diff --git a/numpy/f2py/_backends/src/meson.build.src b/numpy/f2py/_backends/meson.build.template similarity index 100% rename from numpy/f2py/_backends/src/meson.build.src rename to numpy/f2py/_backends/meson.build.template From fa06f8d6ecfa776edb10212a415d72970e272256 Mon Sep 17 00:00:00 2001 From: Rohit Goswami Date: Sat, 26 Aug 2023 10:46:50 +0000 Subject: [PATCH 31/37] ENH: Support build_type via --debug [f2py-meson] --- numpy/f2py/_backends/_meson.py | 8 +++++++- numpy/f2py/_backends/meson.build.template | 8 +++++--- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/numpy/f2py/_backends/_meson.py b/numpy/f2py/_backends/_meson.py index 48130edbe464..5e4e68382459 100644 --- a/numpy/f2py/_backends/_meson.py +++ b/numpy/f2py/_backends/_meson.py @@ -22,6 +22,7 @@ def __init__( object_files: list[Path], linker_args: list[str], c_args: list[str], + build_type: str, ): self.modulename = modulename self.build_template_path = ( @@ -36,6 +37,7 @@ def __init__( self.sources_substitution, self.deps_substitution, ] + self.build_type = build_type def meson_build_template(self) -> str: if not self.build_template_path.is_file(): @@ -48,8 +50,8 @@ def meson_build_template(self) -> str: return self.build_template_path.read_text() def initialize_template(self) -> None: - """Initialize with module name.""" self.substitutions["modulename"] = self.modulename + self.substitutions["buildtype"] = self.build_type def sources_substitution(self) -> None: indent = " " * 21 @@ -80,6 +82,9 @@ def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.dependencies = self.extra_dat.get("dependencies", []) self.meson_build_dir = "bbdir" + self.build_type = ( + "debug" if any("debug" in flag for flag in self.fc_flags) else "release" + ) def _move_exec_to_root(self, build_dir: Path): walk_dir = Path(build_dir) / self.meson_build_dir @@ -103,6 +108,7 @@ def write_meson_build(self, build_dir: Path) -> None: self.extra_objects, self.flib_flags, self.fc_flags, + self.build_type, ) src = meson_template.generate_meson_build() Path(build_dir).mkdir(parents=True, exist_ok=True) diff --git a/numpy/f2py/_backends/meson.build.template b/numpy/f2py/_backends/meson.build.template index 29d711f7d37f..545e3995218a 100644 --- a/numpy/f2py/_backends/meson.build.template +++ b/numpy/f2py/_backends/meson.build.template @@ -2,10 +2,12 @@ project('${modulename}', ['c', 'fortran'], version : '0.1', meson_version: '>= 1.1.0', - default_options : ['warning_level=2']) + default_options : [ + 'warning_level=1', + 'buildtype=${buildtype}' + ]) -py_mod = import('python') -py = py_mod.find_installation(pure: false) +py = import('python').find_installation(pure: false) py_dep = py.dependency() incdir_numpy = run_command(py, From 8841647b368d6125624162a564ea7f32d9c8624c Mon Sep 17 00:00:00 2001 From: Rohit Goswami Date: Sat, 26 Aug 2023 10:58:12 +0000 Subject: [PATCH 32/37] MAINT,DOC: Reduce warn,rework doc [f2py-meson] Co-authored-by: rgommers --- doc/release/upcoming_changes/24532.new_feature.rst | 9 ++++++++- numpy/f2py/_backends/_distutils.py | 5 ++--- numpy/f2py/_backends/_meson.py | 5 ----- numpy/f2py/f2py2e.py | 6 ------ 4 files changed, 10 insertions(+), 15 deletions(-) diff --git a/doc/release/upcoming_changes/24532.new_feature.rst b/doc/release/upcoming_changes/24532.new_feature.rst index 6c89d9dd7574..504b1d431cff 100644 --- a/doc/release/upcoming_changes/24532.new_feature.rst +++ b/doc/release/upcoming_changes/24532.new_feature.rst @@ -1,5 +1,12 @@ ``meson`` backend for ``f2py`` ------------------------------ -``f2py`` now accepts the ``--backend meson`` option. This is the default option +``f2py`` in compile mode (i.e. ``f2py -c``) now accepts the ``--backend meson`` option. This is the default option for Python ``3.12`` on-wards. Older versions will still default to ``--backend distutils``. + +To support this in realistic use-cases, in compile mode ``f2py`` takes a +``--dep`` flag one or many times which maps to ``dependency()`` calls in the +``meson`` backend, and does nothing in the ``distutils`` backend. + + +There are no changes for users of ``f2py`` only as a code generator, i.e. without ``-c``. diff --git a/numpy/f2py/_backends/_distutils.py b/numpy/f2py/_backends/_distutils.py index efc7a98b7692..e548fc543010 100644 --- a/numpy/f2py/_backends/_distutils.py +++ b/numpy/f2py/_backends/_distutils.py @@ -13,8 +13,8 @@ class DistutilsBackend(Backend): def __init__(sef, *args, **kwargs): warnings.warn( - "distutils has been deprecated since NumPy 2.16." - "Use the MesonBackend instead, or generate wrappers" + "distutils has been deprecated since NumPy 1.26." + "Use the Meson backend instead, or generate wrappers" "without -c and use a custom build script", VisibleDeprecationWarning, stacklevel=2, @@ -33,7 +33,6 @@ def compile(self): "libraries": self.libraries, "define_macros": self.define_macros, "undef_macros": self.undef_macros, - # TODO: Handle extra_objects in meson "extra_objects": self.extra_objects, "f2py_options": self.f2py_flags, } diff --git a/numpy/f2py/_backends/_meson.py b/numpy/f2py/_backends/_meson.py index 5e4e68382459..3176a5e08f30 100644 --- a/numpy/f2py/_backends/_meson.py +++ b/numpy/f2py/_backends/_meson.py @@ -74,11 +74,6 @@ def generate_meson_build(self): class MesonBackend(Backend): def __init__(self, *args, **kwargs): - warnings.warn( - "The MesonBackend is experimental and may change in the future." - "--build-dir should be used to customize the generated skeleton.", - stacklevel=2, - ) super().__init__(*args, **kwargs) self.dependencies = self.extra_dat.get("dependencies", []) self.meson_build_dir = "bbdir" diff --git a/numpy/f2py/f2py2e.py b/numpy/f2py/f2py2e.py index 3a6a9c429726..95638a880314 100755 --- a/numpy/f2py/f2py2e.py +++ b/numpy/f2py/f2py2e.py @@ -724,12 +724,6 @@ def main(): pass if '-c' in sys.argv[1:]: - import warnings - warnings.warn('Using f2py to compile code without generating files ' - 'first was largely handled by distutils which has been removed in Python 3.12.' - 'The meson replacement is a work in progress, consider running without -c.' - 'Recall that the build can be customized with --build-dir', - stacklevel=2) run_compile() else: run_main(sys.argv[1:]) From 8f214a0118ef3ac0c7739706753663923086b302 Mon Sep 17 00:00:00 2001 From: Rohit Goswami Date: Sat, 26 Aug 2023 11:08:32 +0000 Subject: [PATCH 33/37] ENH: Rework deps: to --dep calls [f2py-meson] Also shows how incremental updates to the parser can be done. --- numpy/f2py/f2py2e.py | 42 +++++++++++++++++++++--------------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/numpy/f2py/f2py2e.py b/numpy/f2py/f2py2e.py index 95638a880314..2422c61a1f45 100755 --- a/numpy/f2py/f2py2e.py +++ b/numpy/f2py/f2py2e.py @@ -20,6 +20,7 @@ import re from pathlib import Path from itertools import dropwhile +import argparse from . import crackfortran from . import rules @@ -143,20 +144,16 @@ --noopt Compile without optimization --noarch Compile without arch-dependent optimization --debug Compile with debugging information - deps: ... : - Specify additional dependencies that the module relies on. - List dependencies separated by spaces, and use `:' to - signify the end of the dependency list. Dependencies are - stored in a list for further processing. + --dep + Specify a meson dependencies for the module. This may + be passed multiple times for multiple dependencies. + Dependencies are stored in a list for further processing. - Example: deps: lapack scalapack : + Example: --dep lapack --dep scalapack This will identify "lapack" and "scalapack" as dependencies and remove them from argv, leaving a dependencies list containing ["lapack", "scalapack"]. - Note: The trailing `:' is required to signify the end of - the dependency list. This is only relevant for the meson backend. - Extra options (only effective with -c): --link- Link extension module with as defined @@ -518,6 +515,17 @@ def get_prefix(module): p = os.path.dirname(os.path.dirname(module.__file__)) return p +def preparse_sysargv(): + # To keep backwards bug compatibility, newer flags are handled by argparse, + # and `sys.argv` is passed to the rest of `f2py` as is. + parser = argparse.ArgumentParser(add_help=False) + parser.add_argument("--dep", action="append", dest="dependencies") + args, remaining_argv = parser.parse_known_args() + sys.argv = [sys.argv[0]] + remaining_argv + return { + "dependencies": args.dependencies or [], + } + def run_compile(): """ @@ -525,6 +533,10 @@ def run_compile(): """ import tempfile + # Collect dependency flags, preprocess sys.argv + argparse_res = preparse_sysargv() + dependencies = argparse_res["dependencies"] + i = sys.argv.index('-c') del sys.argv[i] @@ -563,18 +575,6 @@ def run_compile(): if f2py_flags2 and f2py_flags2[-1] != ':': f2py_flags2.append(':') f2py_flags.extend(f2py_flags2) - # Find the start and end indices of the "deps:" section - start_idx = next((i for i, a in enumerate(sys.argv) if a == "deps:"), None) - end_idx = next((i for i, a in enumerate(sys.argv) if a == ":" and i > start_idx), None) if start_idx is not None else None - # Extract dependencies and remove them from sys.argv - if start_idx is not None and end_idx is not None: - dependencies = sys.argv[start_idx + 1:end_idx] - del sys.argv[start_idx:end_idx + 1] - elif start_idx is not None: - dependencies = list(dropwhile(lambda x: x != ":", sys.argv[start_idx + 1:])) - del sys.argv[start_idx:] - else: - dependencies = [] sys.argv = [_m for _m in sys.argv if _m not in f2py_flags2] _reg3 = re.compile( r'--((f(90)?compiler(-exec|)|compiler)=|help-compiler)') From bc376848faddfce208bf5d0143debb1496b0e062 Mon Sep 17 00:00:00 2001 From: Rohit Goswami Date: Sat, 26 Aug 2023 11:13:22 +0000 Subject: [PATCH 34/37] MAINT,DOC: Add --backend to argparse, add docs --- numpy/f2py/f2py2e.py | 39 ++++++++++++++++++++++----------------- 1 file changed, 22 insertions(+), 17 deletions(-) diff --git a/numpy/f2py/f2py2e.py b/numpy/f2py/f2py2e.py index 2422c61a1f45..1cfe8cddd68c 100755 --- a/numpy/f2py/f2py2e.py +++ b/numpy/f2py/f2py2e.py @@ -144,8 +144,9 @@ --noopt Compile without optimization --noarch Compile without arch-dependent optimization --debug Compile with debugging information + --dep - Specify a meson dependencies for the module. This may + Specify a meson dependency for the module. This may be passed multiple times for multiple dependencies. Dependencies are stored in a list for further processing. @@ -154,6 +155,12 @@ and remove them from argv, leaving a dependencies list containing ["lapack", "scalapack"]. + --backend + Specify the build backend for the compilation process. + The supported backends are 'meson' and 'distutils'. + If not specified, defaults to 'distutils'. On + Python 3.12 or higher, the default is 'meson'. + Extra options (only effective with -c): --link- Link extension module with as defined @@ -520,13 +527,21 @@ def preparse_sysargv(): # and `sys.argv` is passed to the rest of `f2py` as is. parser = argparse.ArgumentParser(add_help=False) parser.add_argument("--dep", action="append", dest="dependencies") + parser.add_argument("--backend", choices=['meson', 'distutils'], default='distutils') + args, remaining_argv = parser.parse_known_args() sys.argv = [sys.argv[0]] + remaining_argv + + backend_key = args.backend + if sys.version_info >= (3, 12) and backend_key == 'distutils': + outmess('Cannot use distutils backend with Python 3.12, using meson backend instead.') + backend_key = 'meson' + return { "dependencies": args.dependencies or [], + "backend": backend_key } - def run_compile(): """ Do it all in one call! @@ -534,8 +549,11 @@ def run_compile(): import tempfile # Collect dependency flags, preprocess sys.argv - argparse_res = preparse_sysargv() - dependencies = argparse_res["dependencies"] + argy = preparse_sysargv() + dependencies = argy["dependencies"] + backend_key = argy["backend"] + build_backend = f2py_build_generator(backend_key) + i = sys.argv.index('-c') del sys.argv[i] @@ -616,19 +634,6 @@ def run_compile(): if '--quiet' in f2py_flags: setup_flags.append('--quiet') - # Check for and remove --backend first - if '--backend' in sys.argv: - backend_index = sys.argv.index('--backend') - backend_key = sys.argv.pop(backend_index + 1) - sys.argv.pop(backend_index) - else: - backend_key = 'distutils' - - if sys.version_info >= (3, 12) and backend_key == 'distutils': - outmess('Cannot use distutils backend with Python 3.12, using meson backend instead.') - backend_key = 'meson' - build_backend = f2py_build_generator(backend_key) - modulename = 'untitled' sources = sys.argv[1:] From 4e3336b93ed1c00f06fb90c8148c8b071851657a Mon Sep 17 00:00:00 2001 From: Rohit Goswami Date: Sat, 26 Aug 2023 11:33:33 +0000 Subject: [PATCH 35/37] MAINT: Rename meson template [f2py-meson] --- numpy/f2py/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/numpy/f2py/setup.py b/numpy/f2py/setup.py index 075d4683699e..98f1e9aaae84 100644 --- a/numpy/f2py/setup.py +++ b/numpy/f2py/setup.py @@ -31,7 +31,7 @@ def configuration(parent_package='', top_path=None): config.add_data_files( 'src/fortranobject.c', 'src/fortranobject.h', - 'backends/src/meson.build.src', + 'backends/meson.build.template', ) config.add_data_files('*.pyi') return config From 518074eb5d3b449d5a6b510a23b1b955684ea89a Mon Sep 17 00:00:00 2001 From: Rohit Goswami Date: Tue, 29 Aug 2023 12:39:47 +0000 Subject: [PATCH 36/37] MAINT: Add meson.build for f2py Should address https://github.com/numpy/numpy/pull/22225#issuecomment-1697208937 --- numpy/f2py/meson.build | 14 ++++++++++++++ numpy/meson.build | 1 + 2 files changed, 15 insertions(+) create mode 100644 numpy/f2py/meson.build diff --git a/numpy/f2py/meson.build b/numpy/f2py/meson.build new file mode 100644 index 000000000000..7a116faf5222 --- /dev/null +++ b/numpy/f2py/meson.build @@ -0,0 +1,14 @@ +py.install_sources( + [ + 'src/fortranobject.c', + 'src/fortranobject.h', + ], + subdir: 'numpy/f2py/src' +) + +py.install_sources( + [ + '_backends/meson.build.template', + ], + subdir: 'numpy/f2py/_backends' +) diff --git a/numpy/meson.build b/numpy/meson.build index a4509bc33067..f3f29138c954 100644 --- a/numpy/meson.build +++ b/numpy/meson.build @@ -413,3 +413,4 @@ subdir('core') subdir('fft') subdir('linalg') subdir('random') +subdir('f2py') From cc92d2c038c18b16c637a46eef85790734f1c37b Mon Sep 17 00:00:00 2001 From: Ralf Gommers Date: Tue, 5 Sep 2023 17:23:35 +0200 Subject: [PATCH 37/37] BLD: remove duplicate f2py handling in meson.build files --- numpy/f2py/meson.build | 14 -------------- numpy/meson.build | 1 - 2 files changed, 15 deletions(-) delete mode 100644 numpy/f2py/meson.build diff --git a/numpy/f2py/meson.build b/numpy/f2py/meson.build deleted file mode 100644 index 7a116faf5222..000000000000 --- a/numpy/f2py/meson.build +++ /dev/null @@ -1,14 +0,0 @@ -py.install_sources( - [ - 'src/fortranobject.c', - 'src/fortranobject.h', - ], - subdir: 'numpy/f2py/src' -) - -py.install_sources( - [ - '_backends/meson.build.template', - ], - subdir: 'numpy/f2py/_backends' -) diff --git a/numpy/meson.build b/numpy/meson.build index f3f29138c954..a4509bc33067 100644 --- a/numpy/meson.build +++ b/numpy/meson.build @@ -413,4 +413,3 @@ subdir('core') subdir('fft') subdir('linalg') subdir('random') -subdir('f2py')