diff --git a/doc/release/upcoming_changes/24532.new_feature.rst b/doc/release/upcoming_changes/24532.new_feature.rst new file mode 100644 index 000000000000..504b1d431cff --- /dev/null +++ b/doc/release/upcoming_changes/24532.new_feature.rst @@ -0,0 +1,12 @@ +``meson`` backend for ``f2py`` +------------------------------ +``f2py`` in compile mode (i.e. ``f2py -c``) now accepts the ``--backend meson`` option. This is the default option +for Python ``3.12`` on-wards. Older versions will still default to ``--backend +distutils``. + +To support this in realistic use-cases, in compile mode ``f2py`` takes a +``--dep`` flag one or many times which maps to ``dependency()`` calls in the +``meson`` backend, and does nothing in the ``distutils`` backend. + + +There are no changes for users of ``f2py`` only as a code generator, i.e. without ``-c``. diff --git a/numpy/distutils/command/build_src.py b/numpy/distutils/command/build_src.py index bf3d03c70e44..7303db124cc8 100644 --- a/numpy/distutils/command/build_src.py +++ b/numpy/distutils/command/build_src.py @@ -539,8 +539,8 @@ def f2py_sources(self, sources, extension): if (self.force or newer_group(depends, target_file, 'newer')) \ and not skip_f2py: log.info("f2py: %s" % (source)) - import numpy.f2py - numpy.f2py.run_main(f2py_options + from numpy.f2py import f2py2e + f2py2e.run_main(f2py_options + ['--build-dir', target_dir, source]) else: log.debug(" skipping '%s' f2py interface (up-to-date)" % (source)) @@ -558,8 +558,8 @@ def f2py_sources(self, sources, extension): and not skip_f2py: log.info("f2py:> %s" % (target_file)) self.mkpath(target_dir) - import numpy.f2py - numpy.f2py.run_main(f2py_options + ['--lower', + from numpy.f2py import f2py2e + f2py2e.run_main(f2py_options + ['--lower', '--build-dir', target_dir]+\ ['-m', ext_name]+f_sources) else: diff --git a/numpy/f2py/_backends/__init__.py b/numpy/f2py/_backends/__init__.py new file mode 100644 index 000000000000..e91393c14be3 --- /dev/null +++ b/numpy/f2py/_backends/__init__.py @@ -0,0 +1,9 @@ +def f2py_build_generator(name): + if name == "meson": + from ._meson import MesonBackend + return MesonBackend + elif name == "distutils": + from ._distutils import DistutilsBackend + return DistutilsBackend + else: + raise ValueError(f"Unknown backend: {name}") diff --git a/numpy/f2py/_backends/_backend.py b/numpy/f2py/_backends/_backend.py new file mode 100644 index 000000000000..a7d43d2587b2 --- /dev/null +++ b/numpy/f2py/_backends/_backend.py @@ -0,0 +1,46 @@ +from __future__ import annotations + +from abc import ABC, abstractmethod + + +class Backend(ABC): + def __init__( + self, + modulename, + sources, + extra_objects, + build_dir, + include_dirs, + library_dirs, + libraries, + define_macros, + undef_macros, + f2py_flags, + sysinfo_flags, + fc_flags, + flib_flags, + setup_flags, + remove_build_dir, + extra_dat, + ): + self.modulename = modulename + self.sources = sources + self.extra_objects = extra_objects + self.build_dir = build_dir + self.include_dirs = include_dirs + self.library_dirs = library_dirs + self.libraries = libraries + self.define_macros = define_macros + self.undef_macros = undef_macros + self.f2py_flags = f2py_flags + self.sysinfo_flags = sysinfo_flags + self.fc_flags = fc_flags + self.flib_flags = flib_flags + self.setup_flags = setup_flags + self.remove_build_dir = remove_build_dir + self.extra_dat = extra_dat + + @abstractmethod + def compile(self) -> None: + """Compile the wrapper.""" + pass diff --git a/numpy/f2py/_backends/_distutils.py b/numpy/f2py/_backends/_distutils.py new file mode 100644 index 000000000000..e548fc543010 --- /dev/null +++ b/numpy/f2py/_backends/_distutils.py @@ -0,0 +1,75 @@ +from ._backend import Backend + +from numpy.distutils.core import setup, Extension +from numpy.distutils.system_info import get_info +from numpy.distutils.misc_util import dict_append +from numpy.exceptions import VisibleDeprecationWarning +import os +import sys +import shutil +import warnings + + +class DistutilsBackend(Backend): + def __init__(sef, *args, **kwargs): + warnings.warn( + "distutils has been deprecated since NumPy 1.26." + "Use the Meson backend instead, or generate wrappers" + "without -c and use a custom build script", + VisibleDeprecationWarning, + stacklevel=2, + ) + super().__init__(*args, **kwargs) + + def compile(self): + num_info = {} + if num_info: + self.include_dirs.extend(num_info.get("include_dirs", [])) + ext_args = { + "name": self.modulename, + "sources": self.sources, + "include_dirs": self.include_dirs, + "library_dirs": self.library_dirs, + "libraries": self.libraries, + "define_macros": self.define_macros, + "undef_macros": self.undef_macros, + "extra_objects": self.extra_objects, + "f2py_options": self.f2py_flags, + } + + if self.sysinfo_flags: + for n in self.sysinfo_flags: + i = get_info(n) + if not i: + print( + f"No {repr(n)} resources found" + "in system (try `f2py --help-link`)" + ) + dict_append(ext_args, **i) + + ext = Extension(**ext_args) + + sys.argv = [sys.argv[0]] + self.setup_flags + sys.argv.extend( + [ + "build", + "--build-temp", + self.build_dir, + "--build-base", + self.build_dir, + "--build-platlib", + ".", + "--disable-optimization", + ] + ) + + if self.fc_flags: + sys.argv.extend(["config_fc"] + self.fc_flags) + if self.flib_flags: + sys.argv.extend(["build_ext"] + self.flib_flags) + + setup(ext_modules=[ext]) + + if self.remove_build_dir and os.path.exists(self.build_dir): + print(f"Removing build directory {self.build_dir}") + shutil.rmtree(self.build_dir) diff --git a/numpy/f2py/_backends/_meson.py b/numpy/f2py/_backends/_meson.py new file mode 100644 index 000000000000..3176a5e08f30 --- /dev/null +++ b/numpy/f2py/_backends/_meson.py @@ -0,0 +1,157 @@ +from __future__ import annotations + +import errno +import shutil +import subprocess +from pathlib import Path + +from ._backend import Backend +from string import Template + +import warnings + + +class MesonTemplate: + """Template meson build file generation class.""" + + def __init__( + self, + modulename: str, + sources: list[Path], + deps: list[str], + object_files: list[Path], + linker_args: list[str], + c_args: list[str], + build_type: str, + ): + self.modulename = modulename + self.build_template_path = ( + Path(__file__).parent.absolute() / "meson.build.template" + ) + self.sources = sources + self.deps = deps + self.substitutions = {} + self.objects = object_files + self.pipeline = [ + self.initialize_template, + self.sources_substitution, + self.deps_substitution, + ] + self.build_type = build_type + + def meson_build_template(self) -> str: + if not self.build_template_path.is_file(): + raise FileNotFoundError( + errno.ENOENT, + "Meson build template" + f" {self.build_template_path.absolute()}" + " does not exist.", + ) + return self.build_template_path.read_text() + + def initialize_template(self) -> None: + self.substitutions["modulename"] = self.modulename + self.substitutions["buildtype"] = self.build_type + + def sources_substitution(self) -> None: + indent = " " * 21 + self.substitutions["source_list"] = f",\n{indent}".join( + [f"'{source}'" for source in self.sources] + ) + + def deps_substitution(self) -> None: + indent = " " * 21 + self.substitutions["dep_list"] = f",\n{indent}".join( + [f"dependency('{dep}')" for dep in self.deps] + ) + + def generate_meson_build(self): + for node in self.pipeline: + node() + template = Template(self.meson_build_template()) + return template.substitute(self.substitutions) + + +class MesonBackend(Backend): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.dependencies = self.extra_dat.get("dependencies", []) + self.meson_build_dir = "bbdir" + self.build_type = ( + "debug" if any("debug" in flag for flag in self.fc_flags) else "release" + ) + + def _move_exec_to_root(self, build_dir: Path): + walk_dir = Path(build_dir) / self.meson_build_dir + path_objects = walk_dir.glob(f"{self.modulename}*.so") + for path_object in path_objects: + shutil.move(path_object, Path.cwd()) + + def _get_build_command(self): + return [ + "meson", + "setup", + self.meson_build_dir, + ] + + def write_meson_build(self, build_dir: Path) -> None: + """Writes the meson build file at specified location""" + meson_template = MesonTemplate( + self.modulename, + self.sources, + self.dependencies, + self.extra_objects, + self.flib_flags, + self.fc_flags, + self.build_type, + ) + src = meson_template.generate_meson_build() + Path(build_dir).mkdir(parents=True, exist_ok=True) + meson_build_file = Path(build_dir) / "meson.build" + meson_build_file.write_text(src) + return meson_build_file + + def run_meson(self, build_dir: Path): + completed_process = subprocess.run(self._get_build_command(), cwd=build_dir) + if completed_process.returncode != 0: + raise subprocess.CalledProcessError( + completed_process.returncode, completed_process.args + ) + completed_process = subprocess.run( + ["meson", "compile", "-C", self.meson_build_dir], cwd=build_dir + ) + if completed_process.returncode != 0: + raise subprocess.CalledProcessError( + completed_process.returncode, completed_process.args + ) + + def compile(self) -> None: + self.sources = _prepare_sources(self.modulename, self.sources, self.build_dir) + self.write_meson_build(self.build_dir) + self.run_meson(self.build_dir) + self._move_exec_to_root(self.build_dir) + + +def _prepare_sources(mname, sources, bdir): + extended_sources = sources.copy() + Path(bdir).mkdir(parents=True, exist_ok=True) + # Copy sources + for source in sources: + shutil.copy(source, bdir) + generated_sources = [ + Path(f"{mname}module.c"), + Path(f"{mname}-f2pywrappers2.f90"), + Path(f"{mname}-f2pywrappers.f"), + ] + bdir = Path(bdir) + for generated_source in generated_sources: + if generated_source.exists(): + shutil.copy(generated_source, bdir / generated_source.name) + extended_sources.append(generated_source.name) + generated_source.unlink() + extended_sources = [ + Path(source).name + for source in extended_sources + if not Path(source).suffix == ".pyf" + ] + return extended_sources diff --git a/numpy/f2py/_backends/meson.build.template b/numpy/f2py/_backends/meson.build.template new file mode 100644 index 000000000000..545e3995218a --- /dev/null +++ b/numpy/f2py/_backends/meson.build.template @@ -0,0 +1,42 @@ +project('${modulename}', + ['c', 'fortran'], + version : '0.1', + meson_version: '>= 1.1.0', + default_options : [ + 'warning_level=1', + 'buildtype=${buildtype}' + ]) + +py = import('python').find_installation(pure: false) +py_dep = py.dependency() + +incdir_numpy = run_command(py, + ['-c', 'import os; os.chdir(".."); import numpy; print(numpy.get_include())'], + check : true +).stdout().strip() + +incdir_f2py = run_command(py, + ['-c', 'import os; os.chdir(".."); import numpy.f2py; print(numpy.f2py.get_include())'], + check : true +).stdout().strip() + +inc_np = include_directories(incdir_numpy) +np_dep = declare_dependency(include_directories: inc_np) + +incdir_f2py = incdir_numpy / '..' / '..' / 'f2py' / 'src' +inc_f2py = include_directories(incdir_f2py) +fortranobject_c = incdir_f2py / 'fortranobject.c' + +inc_np = include_directories(incdir_numpy, incdir_f2py) + +py.extension_module('${modulename}', + [ +${source_list}, + fortranobject_c + ], + include_directories: [inc_np], + dependencies : [ + py_dep, +${dep_list} + ], + install : true) diff --git a/numpy/f2py/auxfuncs.py b/numpy/f2py/auxfuncs.py index c0864b5bc613..535e324286bd 100644 --- a/numpy/f2py/auxfuncs.py +++ b/numpy/f2py/auxfuncs.py @@ -16,6 +16,7 @@ """ import pprint import sys +import re import types from functools import reduce from copy import deepcopy @@ -43,7 +44,7 @@ 'ismodule', 'ismoduleroutine', 'isoptional', 'isprivate', 'isrequired', 'isroutine', 'isscalar', 'issigned_long_longarray', 'isstring', 'isstringarray', 'isstring_or_stringarray', 'isstringfunction', - 'issubroutine', + 'issubroutine', 'get_f2py_modulename', 'issubroutine_wrap', 'isthreadsafe', 'isunsigned', 'isunsigned_char', 'isunsigned_chararray', 'isunsigned_long_long', 'isunsigned_long_longarray', 'isunsigned_short', @@ -912,3 +913,20 @@ def deep_merge(dict1, dict2): else: merged_dict[key] = value return merged_dict + +_f2py_module_name_match = re.compile(r'\s*python\s*module\s*(?P[\w_]+)', + re.I).match +_f2py_user_module_name_match = re.compile(r'\s*python\s*module\s*(?P[\w_]*?' + r'__user__[\w_]*)', re.I).match + +def get_f2py_modulename(source): + name = None + with open(source) as f: + for line in f: + m = _f2py_module_name_match(line) + if m: + if _f2py_user_module_name_match(line): # skip *__user__* names + continue + name = m.group('name') + break + return name diff --git a/numpy/f2py/f2py2e.py b/numpy/f2py/f2py2e.py index 10508488dc04..1cfe8cddd68c 100755 --- a/numpy/f2py/f2py2e.py +++ b/numpy/f2py/f2py2e.py @@ -19,6 +19,8 @@ import pprint import re from pathlib import Path +from itertools import dropwhile +import argparse from . import crackfortran from . import rules @@ -28,6 +30,7 @@ from . import f90mod_rules from . import __version__ from . import capi_maps +from numpy.f2py._backends import f2py_build_generator f2py_version = __version__.version numpy_version = __version__.version @@ -126,7 +129,7 @@ -v Print f2py version ID and exit. -numpy.distutils options (only effective with -c): +build backend options (only effective with -c): --fcompiler= Specify Fortran compiler type by vendor --compiler= Specify C compiler type (as defined by distutils) @@ -142,6 +145,22 @@ --noarch Compile without arch-dependent optimization --debug Compile with debugging information + --dep + Specify a meson dependency for the module. This may + be passed multiple times for multiple dependencies. + Dependencies are stored in a list for further processing. + + Example: --dep lapack --dep scalapack + This will identify "lapack" and "scalapack" as dependencies + and remove them from argv, leaving a dependencies list + containing ["lapack", "scalapack"]. + + --backend + Specify the build backend for the compilation process. + The supported backends are 'meson' and 'distutils'. + If not specified, defaults to 'distutils'. On + Python 3.12 or higher, the default is 'meson'. + Extra options (only effective with -c): --link- Link extension module with as defined @@ -251,6 +270,8 @@ def scaninputline(inputline): 'f2py option --include_paths is deprecated, use --include-paths instead.\n') f7 = 1 elif l[:15] in '--include-paths': + # Similar to using -I with -c, however this is + # also used during generation of wrappers f7 = 1 elif l == '--skip-empty-wrappers': emptygen = False @@ -501,6 +522,25 @@ def get_prefix(module): p = os.path.dirname(os.path.dirname(module.__file__)) return p +def preparse_sysargv(): + # To keep backwards bug compatibility, newer flags are handled by argparse, + # and `sys.argv` is passed to the rest of `f2py` as is. + parser = argparse.ArgumentParser(add_help=False) + parser.add_argument("--dep", action="append", dest="dependencies") + parser.add_argument("--backend", choices=['meson', 'distutils'], default='distutils') + + args, remaining_argv = parser.parse_known_args() + sys.argv = [sys.argv[0]] + remaining_argv + + backend_key = args.backend + if sys.version_info >= (3, 12) and backend_key == 'distutils': + outmess('Cannot use distutils backend with Python 3.12, using meson backend instead.') + backend_key = 'meson' + + return { + "dependencies": args.dependencies or [], + "backend": backend_key + } def run_compile(): """ @@ -508,6 +548,13 @@ def run_compile(): """ import tempfile + # Collect dependency flags, preprocess sys.argv + argy = preparse_sysargv() + dependencies = argy["dependencies"] + backend_key = argy["backend"] + build_backend = f2py_build_generator(backend_key) + + i = sys.argv.index('-c') del sys.argv[i] @@ -546,7 +593,6 @@ def run_compile(): if f2py_flags2 and f2py_flags2[-1] != ':': f2py_flags2.append(':') f2py_flags.extend(f2py_flags2) - sys.argv = [_m for _m in sys.argv if _m not in f2py_flags2] _reg3 = re.compile( r'--((f(90)?compiler(-exec|)|compiler)=|help-compiler)') @@ -598,17 +644,17 @@ def run_compile(): del sys.argv[i + 1], sys.argv[i] sources = sys.argv[1:] + pyf_files = [] if '-m' in sys.argv: i = sys.argv.index('-m') modulename = sys.argv[i + 1] del sys.argv[i + 1], sys.argv[i] sources = sys.argv[1:] else: - from numpy.distutils.command.build_src import get_f2py_modulename - pyf_files, sources = filter_files('', '[.]pyf([.]src|)', sources) - sources = pyf_files + sources + pyf_files, _sources = filter_files('', '[.]pyf([.]src|)', sources) + sources = pyf_files + _sources for f in pyf_files: - modulename = get_f2py_modulename(f) + modulename = auxfuncs.get_f2py_modulename(f) if modulename: break @@ -627,52 +673,36 @@ def run_compile(): else: print('Invalid use of -D:', name_value) - from numpy.distutils.system_info import get_info - - num_info = {} - if num_info: - include_dirs.extend(num_info.get('include_dirs', [])) - - from numpy.distutils.core import setup, Extension - ext_args = {'name': modulename, 'sources': sources, - 'include_dirs': include_dirs, - 'library_dirs': library_dirs, - 'libraries': libraries, - 'define_macros': define_macros, - 'undef_macros': undef_macros, - 'extra_objects': extra_objects, - 'f2py_options': f2py_flags, - } - - if sysinfo_flags: - from numpy.distutils.misc_util import dict_append - for n in sysinfo_flags: - i = get_info(n) - if not i: - outmess('No %s resources found in system' - ' (try `f2py --help-link`)\n' % (repr(n))) - dict_append(ext_args, **i) - - ext = Extension(**ext_args) - sys.argv = [sys.argv[0]] + setup_flags - sys.argv.extend(['build', - '--build-temp', build_dir, - '--build-base', build_dir, - '--build-platlib', '.', - # disable CCompilerOpt - '--disable-optimization']) - if fc_flags: - sys.argv.extend(['config_fc'] + fc_flags) - if flib_flags: - sys.argv.extend(['build_ext'] + flib_flags) - - setup(ext_modules=[ext]) - - if remove_build_dir and os.path.exists(build_dir): - import shutil - outmess('Removing build directory %s\n' % (build_dir)) - shutil.rmtree(build_dir) - + # Construct wrappers / signatures / things + if backend_key == 'meson': + outmess('Using meson backend\nWill pass --lower to f2py\nSee https://numpy.org/doc/stable/f2py/buildtools/meson.html') + f2py_flags.append('--lower') + if pyf_files: + run_main(f" {' '.join(f2py_flags)} {' '.join(pyf_files)}".split()) + else: + run_main(f" {' '.join(f2py_flags)} -m {modulename} {' '.join(sources)}".split()) + + # Now use the builder + builder = build_backend( + modulename, + sources, + extra_objects, + build_dir, + include_dirs, + library_dirs, + libraries, + define_macros, + undef_macros, + f2py_flags, + sysinfo_flags, + fc_flags, + flib_flags, + setup_flags, + remove_build_dir, + {"dependencies": dependencies}, + ) + + builder.compile() def main(): if '--help-link' in sys.argv[1:]: diff --git a/numpy/f2py/setup.py b/numpy/f2py/setup.py index 499609f96600..98f1e9aaae84 100644 --- a/numpy/f2py/setup.py +++ b/numpy/f2py/setup.py @@ -26,10 +26,13 @@ def configuration(parent_package='', top_path=None): config = Configuration('f2py', parent_package, top_path) config.add_subpackage('tests') + config.add_subpackage('_backends') config.add_data_dir('tests/src') config.add_data_files( 'src/fortranobject.c', - 'src/fortranobject.h') + 'src/fortranobject.h', + 'backends/meson.build.template', + ) config.add_data_files('*.pyi') return config diff --git a/numpy/f2py/tests/src/f2cmap/isoFortranEnvMap.f90 b/numpy/f2py/tests/src/f2cmap/isoFortranEnvMap.f90 index 3f0e12c76833..1e1dc1d4054b 100644 --- a/numpy/f2py/tests/src/f2cmap/isoFortranEnvMap.f90 +++ b/numpy/f2py/tests/src/f2cmap/isoFortranEnvMap.f90 @@ -4,6 +4,6 @@ subroutine func1(n, x, res) integer(int64), intent(in) :: n real(real64), intent(in) :: x(n) real(real64), intent(out) :: res -Cf2py intent(hide) :: n +!f2py intent(hide) :: n res = sum(x) end