From a5dfd6839140376aef9b54e94d4efb56a79bff85 Mon Sep 17 00:00:00 2001 From: Rohit Goswami Date: Tue, 5 Oct 2021 01:11:43 +0000 Subject: [PATCH 001/132] ENH: Start refactoring f2py front-end --- numpy/f2py/f2pyarg.py | 76 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 76 insertions(+) create mode 100644 numpy/f2py/f2pyarg.py diff --git a/numpy/f2py/f2pyarg.py b/numpy/f2py/f2pyarg.py new file mode 100644 index 000000000000..5705a8298058 --- /dev/null +++ b/numpy/f2py/f2pyarg.py @@ -0,0 +1,76 @@ +#!/usr/bin/env python3 + +""" +argparse front-end to f2py + +The concept is based around the idea that F2PY is overloaded in terms of +functionality: + +1. Creating the wrapper `.c` files +2. Generating `.pyf` signature files +3. Compilation helpers + a. This essentially means `numpy.distutils` for now + +The three functionalities are largely independent of each other, hence the +implementation in terms of subparsers +""" + +import argparse + + +parser = argparse.ArgumentParser( + prog="f2py", + description="""This program generates a Python C/API file + (module.c) that contains wrappers for given fortran + functions so that they can be called from Python. With the -c + option the corresponding extension modules are built.""", + add_help=False, +) + +build_helpers = parser.add_argument_group("build helpers") +generate_wrappers = parser.add_argument_group("wrappers and signature files") + +# Common # +########## + +parser.add_argument( + "Fortran Files", + metavar="", + type=str, + nargs="*", + help="""Paths to fortran/signature files that will be scanned for + in order to determine their signatures.""", +) + +parser.add_argument("--2d-numpy", default=True, help="Use f2py with Numeric support") + +# Wrappers/Signatures # +####################### + +generate_wrappers.add_argument( + # TODO: Seriously consider scrapping this naming convention + "-h", + "--hint-signature", + type=str, + nargs=1, + help="""Write signatures of the fortran routines to file and + exit. You can then edit and use it instead of + . If ==stdout then the signatures + are printed to stdout.""", +) + +parser.add_argument("--help", action="store_true", help="Print the help") + + +def process_args(args): + if args.help: + parser.print_help() + elif getattr(args, "Fortran Files"): + print("BOOM") + else: + parser.print_usage() + + +if __name__ == "__main__": + args = parser.parse_args() + process_args(args) From 1eacfb0b7ac99a13c2c1a0fd4effe42c4fda84f8 Mon Sep 17 00:00:00 2001 From: Rohit Goswami Date: Tue, 5 Oct 2021 01:12:28 +0000 Subject: [PATCH 002/132] ENH: Start using logging --- numpy/f2py/f2pyarg.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/numpy/f2py/f2pyarg.py b/numpy/f2py/f2pyarg.py index 5705a8298058..c828f4f97ea5 100644 --- a/numpy/f2py/f2pyarg.py +++ b/numpy/f2py/f2pyarg.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 """ -argparse front-end to f2py +argparse+logging front-end to f2py The concept is based around the idea that F2PY is overloaded in terms of functionality: @@ -16,6 +16,7 @@ """ import argparse +import logging parser = argparse.ArgumentParser( @@ -72,5 +73,7 @@ def process_args(args): if __name__ == "__main__": + logger = logging.getLogger("f2py_cli") + logger.setLevel(logging.WARNING) args = parser.parse_args() process_args(args) From 173a7bbcd575f0c4c80540ed54b9dd055ded0ff3 Mon Sep 17 00:00:00 2001 From: Rohit Goswami Date: Tue, 5 Oct 2021 01:55:26 +0000 Subject: [PATCH 003/132] ENH: Add more options for the new front-end --- numpy/f2py/f2pyarg.py | 69 +++++++++++++++++++++++++++++++++++-------- 1 file changed, 57 insertions(+), 12 deletions(-) diff --git a/numpy/f2py/f2pyarg.py b/numpy/f2py/f2pyarg.py index c828f4f97ea5..1806b16fca55 100644 --- a/numpy/f2py/f2pyarg.py +++ b/numpy/f2py/f2pyarg.py @@ -17,6 +17,28 @@ import argparse import logging +import pathlib + +########### +# Helpers # +########### + + +def check_fortran(fname): + fpname = pathlib.Path(fname) + if fpname.suffix.lower() in [".f90", ".f", ".f77"]: + return fpname + else: + logger.warning( + """Does not look like a standard fortran file ending in *.f90, *.f or + *.f77, continuing against better judgement""" + ) + return fpname + + +########## +# Parser # +########## parser = argparse.ArgumentParser( @@ -25,7 +47,11 @@ (module.c) that contains wrappers for given fortran functions so that they can be called from Python. With the -c option the corresponding extension modules are built.""", - add_help=False, + add_help=False, # Since -h is taken... + # Format to look like f2py + formatter_class=lambda prog: argparse.HelpFormatter( + prog, max_help_position=100, width=80 + ), ) build_helpers = parser.add_argument_group("build helpers") @@ -34,17 +60,37 @@ # Common # ########## +# --help is still free +parser.add_argument("--help", action="store_true", help="Print the help") + +# TODO: Remove? parser.add_argument( - "Fortran Files", - metavar="", + "--2d-numpy", + type=bool, + default=True, + help="Use f2py with Numeric support [DEFAULT]", +) + +parser.add_argument( + "Fortran Functions", + metavar="", type=str, nargs="*", + default="ALL", + help="""Names of fortran routines for which Python C/API + functions will be generated. Default is all that are found + in .""", +) + +parser.add_argument( + "Fortran Files", + metavar="", + type=check_fortran, # Returns a pathlib.Path + nargs="*", # As many as present, 0 OK help="""Paths to fortran/signature files that will be scanned for in order to determine their signatures.""", ) -parser.add_argument("--2d-numpy", default=True, help="Use f2py with Numeric support") - # Wrappers/Signatures # ####################### @@ -52,16 +98,15 @@ # TODO: Seriously consider scrapping this naming convention "-h", "--hint-signature", - type=str, + type=pathlib.Path, nargs=1, - help="""Write signatures of the fortran routines to file and - exit. You can then edit and use it instead of - . If ==stdout then the signatures - are printed to stdout.""", + help=r""" + Write signatures of the fortran routines to file and exit. You + can then edit and use it instead of . If + ==stdout then the signatures are printed to stdout. + """, ) -parser.add_argument("--help", action="store_true", help="Print the help") - def process_args(args): if args.help: From 52762a966d13f970509c166cd6731e5615a29bde Mon Sep 17 00:00:00 2001 From: Rohit Goswami Date: Tue, 5 Oct 2021 03:09:39 +0000 Subject: [PATCH 004/132] ENH: Half the f2py options --- numpy/f2py/f2pyarg.py | 507 +++++++++++++++++++++++++++++++++++++++++- 1 file changed, 495 insertions(+), 12 deletions(-) diff --git a/numpy/f2py/f2pyarg.py b/numpy/f2py/f2pyarg.py index 1806b16fca55..1819913d2cd2 100644 --- a/numpy/f2py/f2pyarg.py +++ b/numpy/f2py/f2pyarg.py @@ -18,13 +18,56 @@ import argparse import logging import pathlib +import tempfile + +from numpy.version import version as __version__ + + +################## +# Temp Variables # +################## + +# TODO: Kill these np.distutil specific variables +npd_link = ['atlas', 'atlas_threads', 'atlas_blas', 'atlas_blas_threads', + 'lapack_atlas', 'lapack_atlas_threads', 'atlas_3_10', + 'atlas_3_10_threads', 'atlas_3_10_blas', 'atlas_3_10_blas_threads' + 'lapack_atlas_3_10', 'lapack_atlas_3_10_threads', 'flame', 'mkl', + 'openblas', 'openblas_lapack', 'openblas_clapack', 'blis', + 'lapack_mkl', 'blas_mkl', 'accelerate', 'openblas64_', + 'openblas64__lapack', 'openblas_ilp64', 'openblas_ilp64_lapack' + 'x11', 'fft_opt', 'fftw', 'fftw2', 'fftw3', 'dfftw', 'sfftw', + 'fftw_threads', 'dfftw_threads', 'sfftw_threads', 'djbfft', 'blas', + 'lapack', 'lapack_src', 'blas_src', 'numpy', 'f2py', 'Numeric', + 'numeric', 'numarray', 'numerix', 'lapack_opt', 'lapack_ilp64_opt', + 'lapack_ilp64_plain_opt', 'lapack64__opt', 'blas_opt', + 'blas_ilp64_opt', 'blas_ilp64_plain_opt', 'blas64__opt', + 'boost_python', 'agg2', 'wx', 'gdk_pixbuf_xlib_2', + 'gdk-pixbuf-xlib-2.0', 'gdk_pixbuf_2', 'gdk-pixbuf-2.0', 'gdk', + 'gdk_2', 'gdk-2.0', 'gdk_x11_2', 'gdk-x11-2.0', 'gtkp_x11_2', + 'gtk+-x11-2.0', 'gtkp_2', 'gtk+-2.0', 'xft', 'freetype2', 'umfpack', + 'amd'] ########### # Helpers # ########### -def check_fortran(fname): +def check_fortran(fname: str): + """Function which checks + + This is meant as a sanity check, but will not raise an error, just a + warning. It is called with ``type`` + + Parameters + ---------- + fname : str + The name of the file + + Returns + ------- + pathlib.Path + This is the string as a path, irrespective of the suffix + """ fpname = pathlib.Path(fname) if fpname.suffix.lower() in [".f90", ".f", ".f77"]: return fpname @@ -36,6 +79,112 @@ def check_fortran(fname): return fpname +def check_dir(dname: str): + """Function which checks the build directory + + This is meant to ensure no odd directories are passed, it will fail if a + file is passed + + Parameters + ---------- + dname : str + The name of the directory, by default it will be a temporary one + + Returns + ------- + pathlib.Path + This is the string as a path + """ + if dname == "tempfile.mkdtemp()": + dname = tempfile.mkdtemp() + return pathlib.Path(dname) + else: + dpname = pathlib.Path(dname) + if dpname.is_dir(): + return dpname + else: + raise RuntimeError(f"{dpname} is not a directory") + + +def check_dccomp(opt: str): + """Function which checks for an np.distutils compliant c compiler + + Meant to enforce sanity checks, note that this just checks against distutils.show_compilers() + + Parameters + ---------- + opt: str + The compiler name, must be a distutils option + + Returns + ------- + str + This is the option as a string + """ + cchoices = ["bcpp", "cygwin", "mingw32", "msvc", "unix"] + if opt in cchoices: + return opt + else: + raise RuntimeError(f"{opt} is not an distutils supported C compiler, choose from {cchoices}") + + +def check_npfcomp(opt: str): + """Function which checks for an np.distutils compliant fortran compiler + + Meant to enforce sanity checks + + Parameters + ---------- + opt: str + The compiler name, must be a np.distutils option + + Returns + ------- + str + This is the option as a string + """ + from numpy.distutils import fcompiler + fcompiler.load_all_fcompiler_classes() + fchoices = list(fcompiler.fcompiler_class.keys()) + if opt in fchoices[0]: + return opt + else: + raise RuntimeError(f"{opt} is not an np.distutils supported compiler, choose from {fchoices}") + + +# TODO: Compatibility helper, kill later +# From 3.9 onwards should be argparse.BooleanOptionalAction +class BoolAction(argparse.Action): + """A custom action to mimic Ruby's --[no]-blah functionality in f2py + + This is meant to use ``argparse`` with a custom action to ensure backwards + compatibility with ``f2py``. Kanged `from here`_. + + .. note:: + + Like in the old ``f2py``, it is not an error to pass both variants of + the flag, the last one will be used + + .. from here: + https://thisdataguy.com/2017/07/03/no-options-with-argparse-and-python/ + """ + + def __init__(self, option_strings, dest, nargs=None, **kwargs): + """Initialization of the boolean flag + + Mimics the parent + """ + super(BoolAction, self).__init__(option_strings, dest, nargs=0, **kwargs) + + def __call__(self, parser, namespace, values, option_string=None): + """The logical negation action + + Essentially this returns the semantically valid operation implied by + --no + """ + setattr(namespace, self.dest, False if option_string.contains("no") else True) + + ########## # Parser # ########## @@ -43,18 +192,42 @@ def check_fortran(fname): parser = argparse.ArgumentParser( prog="f2py", - description="""This program generates a Python C/API file - (module.c) that contains wrappers for given fortran - functions so that they can be called from Python. With the -c - option the corresponding extension modules are built.""", + description=""" + This program generates a Python C/API file (module.c) that + contains wrappers for given fortran functions so that they can be called + from Python. + + With the -c option the corresponding extension modules are built.""", add_help=False, # Since -h is taken... # Format to look like f2py - formatter_class=lambda prog: argparse.HelpFormatter( - prog, max_help_position=100, width=80 + formatter_class=lambda prog: argparse.RawDescriptionHelpFormatter( + prog, max_help_position=100, width=85 ), + epilog=f""" + Using the following macros may be required with non-gcc Fortran + compilers: + -DPREPEND_FORTRAN -DNO_APPEND_FORTRAN -DUPPERCASE_FORTRAN + -DUNDERSCORE_G77 + + When using -DF2PY_REPORT_ATEXIT, a performance report of F2PY + interface is printed out at exit (platforms: Linux). + + When using -DF2PY_REPORT_ON_ARRAY_COPY=, a message is + sent to stderr whenever F2PY interface makes a copy of an + array. Integer sets the threshold for array sizes when + a message should be shown. + + Version: {__version__} + numpy Version: {__version__} + Requires: Python 3.5 or higher. + License: NumPy license (see LICENSE.txt in the NumPy source code) + Copyright 1999 - 2011 Pearu Peterson all rights reserved. + https://web.archive.org/web/20140822061353/http://cens.ioc.ee/projects/f2py2e + """ ) -build_helpers = parser.add_argument_group("build helpers") +# subparsers = parser.add_subparsers(help="Functional subsets") +build_helpers = parser.add_argument_group("build helpers, only with -c") generate_wrappers = parser.add_argument_group("wrappers and signature files") # Common # @@ -75,7 +248,7 @@ def check_fortran(fname): "Fortran Functions", metavar="", type=str, - nargs="*", + nargs="*", # As many as present, 0 OK default="ALL", help="""Names of fortran routines for which Python C/API functions will be generated. Default is all that are found @@ -85,28 +258,338 @@ def check_fortran(fname): parser.add_argument( "Fortran Files", metavar="", + action="extend", # List storage + nargs="*", type=check_fortran, # Returns a pathlib.Path - nargs="*", # As many as present, 0 OK help="""Paths to fortran/signature files that will be scanned for in order to determine their signatures.""", ) +parser.add_argument( + "Skip functions", + metavar="skip:", + action="extend", + type=str, + nargs="*", + help="Ignore fortran functions that follow until `:'.", +) + +parser.add_argument( + "Keep functions", + metavar="only:", + action="extend", + type=str, + nargs="*", + help="Use only fortran functions that follow until `:'.", +) + +# TODO: Remove? +parser.add_argument( + "Fortran Files Again", + metavar=":", + action="extend", + type=check_fortran, + nargs="*", + help="Get back to mode.", +) + +parser.add_argument( + "-m", + "--module", + metavar="", + type=str, + nargs=1, + default="untitled", + help="""Name of the module; f2py generates a Python/C API + file module.c or extension module . + Default is 'untitled'.""", +) + +parser.add_argument( + "--lower", + "--no-lower", + metavar="--[no-]lower", + action=BoolAction, + default=True, + type=bool, + help="""Do [not] lower the cases in . + By default, --lower is assumed with -h key, and --no-lower without -h + key.""", +) + +parser.add_argument( + "-b", + "--build-dir", + metavar="", + type=check_dir, + nargs=1, + default="tempfile.mkdtemp()", + help="""All f2py generated files are created in . + Default is tempfile.mkdtemp().""", +) + +parser.add_argument( + "-o", + "--overwrite-signature", + action="store_true", + help="Overwrite existing signature file.", +) + +parser.add_argument( + "--latex-doc", + "--no-latex-doc", + metavar="--[no-]latex-doc", + action=BoolAction, + type=bool, + default=False, + nargs=1, + help="""Create (or not) module.tex. + Default is --no-latex-doc.""", +) + +parser.add_argument( + "--short-latex", + action="store_true", + help="""Create 'incomplete' LaTeX document (without commands + \\documentclass, \\tableofcontents, and \\begin{{document}}, + \\end{{document}}).""", +) + +parser.add_argument( + "--rest-doc", + "--no-rest-doc", + metavar="--[no-]rest-doc", + action=BoolAction, + type=bool, + default=False, + nargs=1, + help="""Create (or not) module.rst. + Default is --no-rest-doc.""", +) + +parser.add_argument( + "--debug-capi", + action="store_true", + help="""Create C/API code that reports the state of the wrappers + during runtime. Useful for debugging.""", +) + +parser.add_argument( + "--wrap-functions", + "--no-wrap-functions", + metavar="--[no-]wrap-functions", + action=BoolAction, + type=bool, + default=False, + nargs=1, + help="""Create (or not) module.rst. + Default is --no-rest-doc.""", +) + +parser.add_argument( + "--include-paths", + metavar=":", + action="extend", + nargs="*", + type=pathlib.Path, + help="Search include files from the given directories.", +) + +parser.add_argument( + "--help-link", + metavar="..", + action="extend", + nargs="*", + choices=npd_link, + type=str, + help="""List system resources found by system_info.py. See also + --link- switch below. [..] is optional list + of resources names. E.g. try 'f2py --help-link lapack_opt'.""" +) + +parser.add_argument( + "--f2cmap", + metavar="", + action="extend", + nargs="*", + type=pathlib.Path, + default=".f2py_f2cmap", + help="""Load Fortran-to-Python KIND specification from the given + file. Default: .f2py_f2cmap in current directory.""", +) + +parser.add_argument( + "--quiet", + action="store_true", + help="Run quietly.", +) + +parser.add_argument( + "--verbose", + action="store_true", + help="Run with extra verbosity.", +) + +parser.add_argument( + "-v", + action="store_true", + help="Print f2py version ID and exit.", +) + # Wrappers/Signatures # ####################### -generate_wrappers.add_argument( +parser.add_argument( # TODO: Seriously consider scrapping this naming convention "-h", "--hint-signature", + metavar="", type=pathlib.Path, nargs=1, - help=r""" + help=""" Write signatures of the fortran routines to file and exit. You can then edit and use it instead of . If ==stdout then the signatures are printed to stdout. """, ) +# NumPy Distutils # +################### + +# TODO: Generalize to allow -c to take other build systems with numpy.distutils +# as a default +build_helpers.add_argument( + "-c", + default="numpy.distutils", + action="store_true", + help="Compilation (via NumPy distutils)" +) + +build_helpers.add_argument( + "--fcompiler", + nargs=1, + type=check_npfcomp, + help="Specify Fortran compiler type by vendor" +) + +build_helpers.add_argument( + "--compiler", + nargs=1, + type=check_dccomp, + help="Specify distutils C compiler type" +) + +build_helpers.add_argument( + "--help-fcompiler", + action="store_true", + help="List available Fortran compilers and exit" +) + +build_helpers.add_argument( + "--f77exec", + nargs=1, + type=pathlib.Path, + help="Specify the path to a F77 compiler" +) + +build_helpers.add_argument( + "--f90exec", + nargs=1, + type=pathlib.Path, + help="Specify the path to a F90 compiler" +) + +build_helpers.add_argument( + "--f77flags", + nargs="*", + type=pathlib.Path, + action="extend", + help="Specify F77 compiler flags" +) + +build_helpers.add_argument( + "--f90flags", + nargs="*", + type=pathlib.Path, + action="extend", + help="Specify F90 compiler flags" +) + +build_helpers.add_argument( + "--opt", + "--optimization_flags", + nargs="*", + type=str, + action="extend", + help="Specify optimization flags" +) + +build_helpers.add_argument( + "--arch", + "--architecture_optimizations", + nargs="*", + type=str, + action="extend", + help="Specify architecture specific optimization flags" +) + +build_helpers.add_argument( + "--noopt", + action="store_true", + help="Compile without optimization" +) + +build_helpers.add_argument( + "--noarch", + action="store_true", + help="Compile without arch-dependent optimization" +) + +build_helpers.add_argument( + "--debug", + action="store_true", + help="Compile with debugging information" +) + +build_helpers.add_argument( + "-L", + "--library-path", + type=pathlib.Path, + metavar="/path/to/lib/", + nargs="*", + action="extend", + help="Path to library" +) + +build_helpers.add_argument( + "-l", + "--library_name", + type=pathlib.Path, + metavar="", + nargs="*", + action="extend", + help="Library name" +) + +build_helpers.add_argument( + "-I", + "--include_dirs", + type=pathlib.Path, + metavar="/path/to/include", + nargs="*", + action="extend", + help="Include directories" +) + +# TODO: Figure out equivalence +# .o .so .a +# Also, +# build_helpers.add_argument( +# "-l", +# "--link-" +# ) +# ) + def process_args(args): if args.help: From c0689cb6cb7554c7ae6e999546878572da096f4e Mon Sep 17 00:00:00 2001 From: Rohit Goswami Date: Mon, 18 Oct 2021 19:12:28 +0000 Subject: [PATCH 005/132] ENH: Switch to using f2pyarg --- numpy/f2py/__init__.py | 9 ++++----- numpy/f2py/__main__.py | 2 +- numpy/f2py/diagnose.py | 14 +++++++------- numpy/f2py/f2pyarg.py | 16 +++++++++++++++- setup.py | 8 ++++---- 5 files changed, 31 insertions(+), 18 deletions(-) diff --git a/numpy/f2py/__init__.py b/numpy/f2py/__init__.py index 84192f7386d1..2eefe2cfb8c3 100644 --- a/numpy/f2py/__init__.py +++ b/numpy/f2py/__init__.py @@ -2,17 +2,16 @@ """Fortran to Python Interface Generator. """ -__all__ = ['run_main', 'compile', 'get_include'] +__all__ = ['main', 'compile', 'get_include'] import sys import subprocess import os -from . import f2py2e +from . import f2pyarg from . import diagnose -run_main = f2py2e.run_main -main = f2py2e.main +main = f2pyarg.main def compile(source, @@ -102,7 +101,7 @@ def compile(source, c = [sys.executable, '-c', - 'import numpy.f2py as f2py2e;f2py2e.main()'] + args + 'import numpy.f2py as f2pyarg;f2pyarg.main()'] + args try: cp = subprocess.run(c, stdout=subprocess.PIPE, stderr=subprocess.PIPE) diff --git a/numpy/f2py/__main__.py b/numpy/f2py/__main__.py index 936a753a2796..1bf2d8da6979 100644 --- a/numpy/f2py/__main__.py +++ b/numpy/f2py/__main__.py @@ -1,5 +1,5 @@ # See: # https://web.archive.org/web/20140822061353/http://cens.ioc.ee/projects/f2py2e -from numpy.f2py.f2py2e import main +from numpy.f2py.f2pyarg import main main() diff --git a/numpy/f2py/diagnose.py b/numpy/f2py/diagnose.py index 21ee399f035f..bbef0227e811 100644 --- a/numpy/f2py/diagnose.py +++ b/numpy/f2py/diagnose.py @@ -35,11 +35,11 @@ def run(): has_newnumpy = 0 try: - from numpy.f2py import f2py2e - has_f2py2e = 1 + from numpy.f2py import f2pyarg + has_f2pyarg = 1 except ImportError: - print('Failed to import f2py2e:', sys.exc_info()[1]) - has_f2py2e = 0 + print('Failed to import f2pyarg:', sys.exc_info()[1]) + has_f2pyarg = 0 try: import numpy.distutils @@ -60,10 +60,10 @@ def run(): print('error:', msg) print('------') - if has_f2py2e: + if has_f2pyarg: try: - print('Found f2py2e version %r in %s' % - (f2py2e.__version__.version, f2py2e.__file__)) + print('Found f2pyarg version %r in %s' % + (f2pyarg.__version__.version, f2pyarg.__file__)) except Exception as msg: print('error:', msg) print('------') diff --git a/numpy/f2py/f2pyarg.py b/numpy/f2py/f2pyarg.py index 1819913d2cd2..246593247e56 100644 --- a/numpy/f2py/f2pyarg.py +++ b/numpy/f2py/f2pyarg.py @@ -22,6 +22,9 @@ from numpy.version import version as __version__ +# F2PY imports +from . import crackfortran +from . import rules ################## # Temp Variables # @@ -596,12 +599,23 @@ def process_args(args): parser.print_help() elif getattr(args, "Fortran Files"): print("BOOM") + if args.c: + if args.fcompiler: + print(f"got {args.fcompiler}") + elif args.compiler: + print(args.compiler) + else: + print("Compilation requested without options, using defaults") else: parser.print_usage() -if __name__ == "__main__": +def main(): logger = logging.getLogger("f2py_cli") logger.setLevel(logging.WARNING) args = parser.parse_args() process_args(args) + + +if __name__ == "__main__": + main() diff --git a/setup.py b/setup.py index 3aa7504b05cb..db637b4f406a 100755 --- a/setup.py +++ b/setup.py @@ -420,13 +420,13 @@ def setup_package(): # The f2py scripts that will be installed if sys.platform == 'win32': f2py_cmds = [ - 'f2py = numpy.f2py.f2py2e:main', + 'f2py = numpy.f2py.f2pyarg:main', ] else: f2py_cmds = [ - 'f2py = numpy.f2py.f2py2e:main', - 'f2py%s = numpy.f2py.f2py2e:main' % sys.version_info[:1], - 'f2py%s.%s = numpy.f2py.f2py2e:main' % sys.version_info[:2], + 'f2py = numpy.f2py.f2pyarg:main', + 'f2py%s = numpy.f2py.f2pyarg:main' % sys.version_info[:1], + 'f2py%s.%s = numpy.f2py.f2pyarg:main' % sys.version_info[:2], ] cmdclass["sdist"] = sdist_checked From 44bec87edaa9221bcf8d324131e97629c0140824 Mon Sep 17 00:00:00 2001 From: Rohit Goswami Date: Mon, 18 Oct 2021 19:12:58 +0000 Subject: [PATCH 006/132] ENH: Add f2py frontend stuff --- numpy/f2py/f2pyarg.py | 138 +++++++++++++++++++++++++++++++++++++++--- 1 file changed, 131 insertions(+), 7 deletions(-) diff --git a/numpy/f2py/f2pyarg.py b/numpy/f2py/f2pyarg.py index 246593247e56..2cb0a2ac472f 100644 --- a/numpy/f2py/f2pyarg.py +++ b/numpy/f2py/f2pyarg.py @@ -188,6 +188,36 @@ def __call__(self, parser, namespace, values, option_string=None): setattr(namespace, self.dest, False if option_string.contains("no") else True) +# TODO: Generalize or kill this np.distutils specific helper action class +class NPDLinkHelper(argparse.Action): + """A custom action to work with f2py's --link-blah + + This is effectively the same as storing help_link + + """ + + def __init__(self, option_strings, dest, nargs=None, **kwargs): + """Initialization of the boolean flag + + Mimics the parent + """ + super(NPDLinkHelper, self).__init__(option_strings, dest, nargs="*", **kwargs) + + def __call__(self, parser, namespace, values, option_string=None): + """The storage action + + Essentially, split the value on -, store in dest + + """ + items = getattr(namespace, self.dest) or [] + outvar = option_string.split("--link-")[1] + if outvar in npd_link: + # replicate the extend functionality + items.extend(outvar) + else: + raise RuntimeError(f"{outvar} is not in {npd_link}") + + ########## # Parser # ########## @@ -564,6 +594,24 @@ def __call__(self, parser, namespace, values, option_string=None): help="Path to library" ) +build_helpers.add_argument( + "-U", + "--undef-macros", + type=str, + nargs="*", + action="extend", + help="Undefined macros" +) + +build_helpers.add_argument( + "-D", + "--def-macros", + type=str, + nargs="*", + action="extend", + help="Defined macros" +) + build_helpers.add_argument( "-l", "--library_name", @@ -584,14 +632,90 @@ def __call__(self, parser, namespace, values, option_string=None): help="Include directories" ) -# TODO: Figure out equivalence +# TODO: Kill this ASAP +# Also collect in to REMAINDER and extract from there +build_helpers.add_argument( + '--link-atlas', '--link-atlas_threads', '--link-atlas_blas', + '--link-atlas_blas_threads', '--link-lapack_atlas', + '--link-lapack_atlas_threads', '--link-atlas_3_10', + '--link-atlas_3_10_threads', '--link-atlas_3_10_blas', + '--link-atlas_3_10_blas_threadslapack_atlas_3_10', + '--link-lapack_atlas_3_10_threads', '--link-flame', '--link-mkl', + '--link-openblas', '--link-openblas_lapack', '--link-openblas_clapack', + '--link-blis', '--link-lapack_mkl', '--link-blas_mkl', '--link-accelerate', + '--link-openblas64_', '--link-openblas64__lapack', '--link-openblas_ilp64', + '--link-openblas_ilp64_lapackx11', '--link-fft_opt', '--link-fftw', + '--link-fftw2', '--link-fftw3', '--link-dfftw', '--link-sfftw', + '--link-fftw_threads', '--link-dfftw_threads', '--link-sfftw_threads', + '--link-djbfft', '--link-blas', '--link-lapack', '--link-lapack_src', + '--link-blas_src', '--link-numpy', '--link-f2py', '--link-Numeric', + '--link-numeric', '--link-numarray', '--link-numerix', '--link-lapack_opt', + '--link-lapack_ilp64_opt', '--link-lapack_ilp64_plain_opt', + '--link-lapack64__opt', '--link-blas_opt', '--link-blas_ilp64_opt', + '--link-blas_ilp64_plain_opt', '--link-blas64__opt', '--link-boost_python', + '--link-agg2', '--link-wx', '--link-gdk_pixbuf_xlib_2', + '--link-gdk-pixbuf-xlib-2.0', '--link-gdk_pixbuf_2', '--link-gdk-pixbuf-2.0', + '--link-gdk', '--link-gdk_2', '--link-gdk-2.0', '--link-gdk_x11_2', + '--link-gdk-x11-2.0', '--link-gtkp_x11_2', '--link-gtk+-x11-2.0', + '--link-gtkp_2', '--link-gtk+-2.0', '--link-xft', '--link-freetype2', + '--link-umfpack', '--link-amd', + metavar="--link-", + dest="link_resource", + nargs="*", + action=NPDLinkHelper, + help="The link helpers for numpy distutils" +) + + +# The rest, only works for files, since we expect: # .o .so .a -# Also, -# build_helpers.add_argument( -# "-l", -# "--link-" -# ) -# ) +parser.add_argument('otherfiles', + type=pathlib.Path, + nargs=argparse.REMAINDER) + +################## +# Parser Actions # +################## + + +def callcrackfortran(files, options): + rules.options = options + crackfortran.debug = options['debug'] + crackfortran.verbose = options['verbose'] + if 'module' in options: + crackfortran.f77modulename = options['module'] + if 'skipfuncs' in options: + crackfortran.skipfuncs = options['skipfuncs'] + if 'onlyfuncs' in options: + crackfortran.onlyfuncs = options['onlyfuncs'] + crackfortran.include_paths[:] = options['include_paths'] + crackfortran.dolowercase = options['do-lower'] + postlist = crackfortran.crackfortran(files) + if 'signsfile' in options: + outmess('Saving signatures to file "%s"\n' % (options['signsfile'])) + pyf = crackfortran.crack2fortran(postlist) + if options['signsfile'][-6:] == 'stdout': + sys.stdout.write(pyf) + else: + with open(options['signsfile'], 'w') as f: + f.write(pyf) + if options["coutput"] is None: + for mod in postlist: + mod["coutput"] = "%smodule.c" % mod["name"] + else: + for mod in postlist: + mod["coutput"] = options["coutput"] + if options["f2py_wrapper_output"] is None: + for mod in postlist: + mod["f2py_wrapper_output"] = "%s-f2pywrappers.f" % mod["name"] + else: + for mod in postlist: + mod["f2py_wrapper_output"] = options["f2py_wrapper_output"] + return postlist + +################ +# Main Process # +################ def process_args(args): From 81b14f43a07a8c03cd2b544729c897523e2671d3 Mon Sep 17 00:00:00 2001 From: Namami Shanker Date: Fri, 1 Jul 2022 00:58:54 +0530 Subject: [PATCH 007/132] FIX: f2pyarg --build-dir create directory if not present --- numpy/f2py/f2pyarg.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/numpy/f2py/f2pyarg.py b/numpy/f2py/f2pyarg.py index 2cb0a2ac472f..4def32d1e450 100644 --- a/numpy/f2py/f2pyarg.py +++ b/numpy/f2py/f2pyarg.py @@ -86,7 +86,7 @@ def check_dir(dname: str): """Function which checks the build directory This is meant to ensure no odd directories are passed, it will fail if a - file is passed + file is passed. Creates directory if not present. Parameters ---------- @@ -103,10 +103,8 @@ def check_dir(dname: str): return pathlib.Path(dname) else: dpname = pathlib.Path(dname) - if dpname.is_dir(): - return dpname - else: - raise RuntimeError(f"{dpname} is not a directory") + dpname.mkdir(parents=True, exist_ok=True) + return dpname def check_dccomp(opt: str): From 7c68149d1c43546dbcf32d9c6c5d4cb97fad4544 Mon Sep 17 00:00:00 2001 From: Namami Shanker Date: Fri, 1 Jul 2022 01:20:53 +0530 Subject: [PATCH 008/132] MAINT: Move f2pyarg helpers to service --- numpy/f2py/f2pyarg.py | 103 +--------------------------------------- numpy/f2py/service.py | 107 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 108 insertions(+), 102 deletions(-) create mode 100644 numpy/f2py/service.py diff --git a/numpy/f2py/f2pyarg.py b/numpy/f2py/f2pyarg.py index 4def32d1e450..90a099589b01 100644 --- a/numpy/f2py/f2pyarg.py +++ b/numpy/f2py/f2pyarg.py @@ -25,6 +25,7 @@ # F2PY imports from . import crackfortran from . import rules +from .service import check_dccomp, check_npfcomp, check_fortran, check_dir ################## # Temp Variables # @@ -50,108 +51,6 @@ 'gtk+-x11-2.0', 'gtkp_2', 'gtk+-2.0', 'xft', 'freetype2', 'umfpack', 'amd'] -########### -# Helpers # -########### - - -def check_fortran(fname: str): - """Function which checks - - This is meant as a sanity check, but will not raise an error, just a - warning. It is called with ``type`` - - Parameters - ---------- - fname : str - The name of the file - - Returns - ------- - pathlib.Path - This is the string as a path, irrespective of the suffix - """ - fpname = pathlib.Path(fname) - if fpname.suffix.lower() in [".f90", ".f", ".f77"]: - return fpname - else: - logger.warning( - """Does not look like a standard fortran file ending in *.f90, *.f or - *.f77, continuing against better judgement""" - ) - return fpname - - -def check_dir(dname: str): - """Function which checks the build directory - - This is meant to ensure no odd directories are passed, it will fail if a - file is passed. Creates directory if not present. - - Parameters - ---------- - dname : str - The name of the directory, by default it will be a temporary one - - Returns - ------- - pathlib.Path - This is the string as a path - """ - if dname == "tempfile.mkdtemp()": - dname = tempfile.mkdtemp() - return pathlib.Path(dname) - else: - dpname = pathlib.Path(dname) - dpname.mkdir(parents=True, exist_ok=True) - return dpname - - -def check_dccomp(opt: str): - """Function which checks for an np.distutils compliant c compiler - - Meant to enforce sanity checks, note that this just checks against distutils.show_compilers() - - Parameters - ---------- - opt: str - The compiler name, must be a distutils option - - Returns - ------- - str - This is the option as a string - """ - cchoices = ["bcpp", "cygwin", "mingw32", "msvc", "unix"] - if opt in cchoices: - return opt - else: - raise RuntimeError(f"{opt} is not an distutils supported C compiler, choose from {cchoices}") - - -def check_npfcomp(opt: str): - """Function which checks for an np.distutils compliant fortran compiler - - Meant to enforce sanity checks - - Parameters - ---------- - opt: str - The compiler name, must be a np.distutils option - - Returns - ------- - str - This is the option as a string - """ - from numpy.distutils import fcompiler - fcompiler.load_all_fcompiler_classes() - fchoices = list(fcompiler.fcompiler_class.keys()) - if opt in fchoices[0]: - return opt - else: - raise RuntimeError(f"{opt} is not an np.distutils supported compiler, choose from {fchoices}") - # TODO: Compatibility helper, kill later # From 3.9 onwards should be argparse.BooleanOptionalAction diff --git a/numpy/f2py/service.py b/numpy/f2py/service.py new file mode 100644 index 000000000000..fed997d2ec6e --- /dev/null +++ b/numpy/f2py/service.py @@ -0,0 +1,107 @@ +import sys +import os +import logging + +from pathlib import Path +from typing import List, Tuple +from . import crackfortran + +logger = logging.getLogger("f2py_cli") +logger.setLevel(logging.WARNING) + +def check_fortran(fname: str): + """Function which checks + + This is meant as a sanity check, but will not raise an error, just a + warning. It is called with ``type`` + + Parameters + ---------- + fname : str + The name of the file + + Returns + ------- + pathlib.Path + This is the string as a path, irrespective of the suffix + """ + fpname = Path(fname) + if fpname.suffix.lower() not in [".f90", ".f", ".f77"]: + logger.warning( + """Does not look like a standard fortran file ending in *.f90, *.f or + *.f77, continuing against better judgement""" + ) + return fpname + + +def check_dir(dname: str): + """Function which checks the build directory + + This is meant to ensure no odd directories are passed, it will fail if a + file is passed. Creates directory if not present. + + Parameters + ---------- + dname : str + The name of the directory, by default it will be a temporary one + + Returns + ------- + pathlib.Path + This is the string as a path + """ + if dname == "tempfile.mkdtemp()": + dname = tempfile.mkdtemp() + return pathlib.Path(dname) + else: + dpname = pathlib.Path(dname) + dpname.mkdir(parents=True, exist_ok=True) + return dpname + + +def check_dccomp(opt: str): + """Function which checks for an np.distutils compliant c compiler + + Meant to enforce sanity checks, note that this just checks against distutils.show_compilers() + + Parameters + ---------- + opt: str + The compiler name, must be a distutils option + + Returns + ------- + str + This is the option as a string + """ + cchoices = ["bcpp", "cygwin", "mingw32", "msvc", "unix"] + if opt in cchoices: + return opt + else: + raise RuntimeError(f"{opt} is not an distutils supported C compiler, choose from {cchoices}") + + +def check_npfcomp(opt: str): + """Function which checks for an np.distutils compliant fortran compiler + + Meant to enforce sanity checks + + Parameters + ---------- + opt: str + The compiler name, must be a np.distutils option + + Returns + ------- + str + This is the option as a string + """ + from numpy.distutils import fcompiler + fcompiler.load_all_fcompiler_classes() + fchoices = list(fcompiler.fcompiler_class.keys()) + if opt in fchoices[0]: + return opt + else: + raise RuntimeError(f"{opt} is not an np.distutils supported compiler, choose from {fchoices}") + + From 9ab806a5f7464047f4157d978eebef7b174c94c1 Mon Sep 17 00:00:00 2001 From: Namami Shanker Date: Fri, 1 Jul 2022 01:30:42 +0530 Subject: [PATCH 009/132] FIX: Fix incorrect imports in f2pyarg service --- numpy/f2py/service.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/numpy/f2py/service.py b/numpy/f2py/service.py index fed997d2ec6e..df9276a405a8 100644 --- a/numpy/f2py/service.py +++ b/numpy/f2py/service.py @@ -1,6 +1,7 @@ import sys import os import logging +import tempfile from pathlib import Path from typing import List, Tuple @@ -52,9 +53,9 @@ def check_dir(dname: str): """ if dname == "tempfile.mkdtemp()": dname = tempfile.mkdtemp() - return pathlib.Path(dname) + return Path(dname) else: - dpname = pathlib.Path(dname) + dpname = Path(dname) dpname.mkdir(parents=True, exist_ok=True) return dpname From c70272c324e36ab705fe8591930755fe45354c1b Mon Sep 17 00:00:00 2001 From: Namami Shanker Date: Fri, 1 Jul 2022 02:04:02 +0530 Subject: [PATCH 010/132] MAINT: f2py check if input f2cmap file exists --- numpy/f2py/capi_maps.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/numpy/f2py/capi_maps.py b/numpy/f2py/capi_maps.py index f07066a09b67..b0022b11747b 100644 --- a/numpy/f2py/capi_maps.py +++ b/numpy/f2py/capi_maps.py @@ -186,8 +186,8 @@ def load_f2cmap_file(f2cmap_file): if f2cmap_file is None: # Default value f2cmap_file = '.f2py_f2cmap' - if not os.path.isfile(f2cmap_file): - return + if not os.path.isfile(f2cmap_file): + return # User defined additions to f2cmap_all. # f2cmap_file must contain a dictionary of dictionaries, only. For From 9fde95331fa487450efc97dfcc389abb8be7bb6d Mon Sep 17 00:00:00 2001 From: Namami Shanker Date: Fri, 1 Jul 2022 02:50:29 +0530 Subject: [PATCH 011/132] MAINT: Fix --wrap-functions docs --- numpy/f2py/f2pyarg.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/numpy/f2py/f2pyarg.py b/numpy/f2py/f2pyarg.py index 90a099589b01..0c066d756091 100644 --- a/numpy/f2py/f2pyarg.py +++ b/numpy/f2py/f2pyarg.py @@ -312,8 +312,9 @@ def __call__(self, parser, namespace, values, option_string=None): type=bool, default=False, nargs=1, - help="""Create (or not) module.rst. - Default is --no-rest-doc.""", + help="""Create (or not) Fortran subroutine wrappers to Fortran 77 + functions. Default is --wrap-functions because it + ensures maximum portability/compiler independence""", ) parser.add_argument( From 4c4b99aeaafd46b5a4b30e0e5a5026a46707a05f Mon Sep 17 00:00:00 2001 From: Namami Shanker Date: Sat, 2 Jul 2022 18:28:37 +0530 Subject: [PATCH 012/132] MAINT: f2pyarg Remove unnecessary flags --- numpy/f2py/f2pyarg.py | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/numpy/f2py/f2pyarg.py b/numpy/f2py/f2pyarg.py index 0c066d756091..93895bedfeca 100644 --- a/numpy/f2py/f2pyarg.py +++ b/numpy/f2py/f2pyarg.py @@ -167,23 +167,6 @@ def __call__(self, parser, namespace, values, option_string=None): parser.add_argument("--help", action="store_true", help="Print the help") # TODO: Remove? -parser.add_argument( - "--2d-numpy", - type=bool, - default=True, - help="Use f2py with Numeric support [DEFAULT]", -) - -parser.add_argument( - "Fortran Functions", - metavar="", - type=str, - nargs="*", # As many as present, 0 OK - default="ALL", - help="""Names of fortran routines for which Python C/API - functions will be generated. Default is all that are found - in .""", -) parser.add_argument( "Fortran Files", From bbb44a7e1fdc0ddd480caa38c48282e66023f27e Mon Sep 17 00:00:00 2001 From: Namami Shanker Date: Sat, 2 Jul 2022 18:29:15 +0530 Subject: [PATCH 013/132] MAINT: f2pyarg remove unnecessary flags --- numpy/f2py/f2pyarg.py | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/numpy/f2py/f2pyarg.py b/numpy/f2py/f2pyarg.py index 93895bedfeca..4c27f3f2279f 100644 --- a/numpy/f2py/f2pyarg.py +++ b/numpy/f2py/f2pyarg.py @@ -196,16 +196,6 @@ def __call__(self, parser, namespace, values, option_string=None): help="Use only fortran functions that follow until `:'.", ) -# TODO: Remove? -parser.add_argument( - "Fortran Files Again", - metavar=":", - action="extend", - type=check_fortran, - nargs="*", - help="Get back to mode.", -) - parser.add_argument( "-m", "--module", From d26bd6a342a5fb95a7afb96795dc9a13f086172a Mon Sep 17 00:00:00 2001 From: Namami Shanker Date: Sat, 2 Jul 2022 18:30:05 +0530 Subject: [PATCH 014/132] MAINT: f2pyarg fix default values --- numpy/f2py/f2pyarg.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/numpy/f2py/f2pyarg.py b/numpy/f2py/f2pyarg.py index 4c27f3f2279f..bc88a7a64389 100644 --- a/numpy/f2py/f2pyarg.py +++ b/numpy/f2py/f2pyarg.py @@ -213,7 +213,7 @@ def __call__(self, parser, namespace, values, option_string=None): "--no-lower", metavar="--[no-]lower", action=BoolAction, - default=True, + default=False, type=bool, help="""Do [not] lower the cases in . By default, --lower is assumed with -h key, and --no-lower without -h @@ -226,7 +226,6 @@ def __call__(self, parser, namespace, values, option_string=None): metavar="", type=check_dir, nargs=1, - default="tempfile.mkdtemp()", help="""All f2py generated files are created in . Default is tempfile.mkdtemp().""", ) @@ -283,7 +282,7 @@ def __call__(self, parser, namespace, values, option_string=None): metavar="--[no-]wrap-functions", action=BoolAction, type=bool, - default=False, + default=True, nargs=1, help="""Create (or not) Fortran subroutine wrappers to Fortran 77 functions. Default is --wrap-functions because it @@ -294,6 +293,7 @@ def __call__(self, parser, namespace, values, option_string=None): "--include-paths", metavar=":", action="extend", + default=[], nargs="*", type=pathlib.Path, help="Search include files from the given directories.", From 9ce21405423c3049971dbd985869b7b1aeb5c360 Mon Sep 17 00:00:00 2001 From: Namami Shanker Date: Sat, 2 Jul 2022 18:31:07 +0530 Subject: [PATCH 015/132] FIX: f2pyarg transfer -h flag to generate_wrappers parser --- numpy/f2py/f2pyarg.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/numpy/f2py/f2pyarg.py b/numpy/f2py/f2pyarg.py index bc88a7a64389..55997319c883 100644 --- a/numpy/f2py/f2pyarg.py +++ b/numpy/f2py/f2pyarg.py @@ -343,7 +343,7 @@ def __call__(self, parser, namespace, values, option_string=None): # Wrappers/Signatures # ####################### -parser.add_argument( +generate_wrappers.add_argument( # TODO: Seriously consider scrapping this naming convention "-h", "--hint-signature", From bcf79029f639fc5b0c3f4edb527e9367562d17d4 Mon Sep 17 00:00:00 2001 From: Namami Shanker Date: Sat, 2 Jul 2022 18:35:57 +0530 Subject: [PATCH 016/132] ENH: f2pyarg add file segregation method --- numpy/f2py/service.py | 33 ++++++++++++++++++++++++++++++++- 1 file changed, 32 insertions(+), 1 deletion(-) diff --git a/numpy/f2py/service.py b/numpy/f2py/service.py index df9276a405a8..cac6cc077afe 100644 --- a/numpy/f2py/service.py +++ b/numpy/f2py/service.py @@ -4,7 +4,9 @@ import tempfile from pathlib import Path -from typing import List, Tuple +from typing import Any, Dict, List, Tuple +from argparse import Namespace + from . import crackfortran logger = logging.getLogger("f2py_cli") @@ -106,3 +108,32 @@ def check_npfcomp(opt: str): raise RuntimeError(f"{opt} is not an np.distutils supported compiler, choose from {fchoices}") + +def segregate_files(files: List[Path]) -> Tuple[List[Path], List[Path], List[Path], List[Path]]: + """ + Segregate files into three groups: + * .f files + * .o files + * others + """ + f77_ext = ('.f', '.for', '.ftn', '.f77') + f90_ext = ('.f90', '.f95', '.f03', '.f08') + out_ext = ('.o', '.out', '.so', '.a') + + f77_files = [] + f90_files = [] + out_files = [] + other_files = [] + + for f in files: + ext = os.path.splitext(f)[1] + if ext in f77_ext: + f77_files.append(f) + elif ext in f90_ext: + f90_files.append(f) + elif ext in out_ext: + out_files.append(f) + else: + other_files.append(f) + + return f77_files, f90_files, out_files, other_files \ No newline at end of file From 568b94dcf4535ffc568c3255434d7c88d71deb3d Mon Sep 17 00:00:00 2001 From: Namami Shanker Date: Sat, 2 Jul 2022 18:41:29 +0530 Subject: [PATCH 017/132] ENH: Create temp dir for null builddir value --- numpy/f2py/f2pyarg.py | 1 + numpy/f2py/service.py | 5 +---- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/numpy/f2py/f2pyarg.py b/numpy/f2py/f2pyarg.py index 55997319c883..1d654c9ad0c1 100644 --- a/numpy/f2py/f2pyarg.py +++ b/numpy/f2py/f2pyarg.py @@ -590,6 +590,7 @@ def callcrackfortran(files, options): def process_args(args): + build_dir = args.build_dir[0] if args.build_dir else tempfile.mkdtemp() if args.help: parser.print_help() elif getattr(args, "Fortran Files"): diff --git a/numpy/f2py/service.py b/numpy/f2py/service.py index cac6cc077afe..fe05bc8a3994 100644 --- a/numpy/f2py/service.py +++ b/numpy/f2py/service.py @@ -53,10 +53,7 @@ def check_dir(dname: str): pathlib.Path This is the string as a path """ - if dname == "tempfile.mkdtemp()": - dname = tempfile.mkdtemp() - return Path(dname) - else: + if dname: dpname = Path(dname) dpname.mkdir(parents=True, exist_ok=True) return dpname From b3d8a2b9ac7cac97e68a691a0ac77f48789003b8 Mon Sep 17 00:00:00 2001 From: Namami Shanker Date: Sat, 2 Jul 2022 18:44:02 +0530 Subject: [PATCH 018/132] ENH: Chop callcrackfortran and move it to service. Add other helpers. --- numpy/f2py/f2pyarg.py | 39 --------------- numpy/f2py/service.py | 113 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 113 insertions(+), 39 deletions(-) diff --git a/numpy/f2py/f2pyarg.py b/numpy/f2py/f2pyarg.py index 1d654c9ad0c1..0b2388858bd9 100644 --- a/numpy/f2py/f2pyarg.py +++ b/numpy/f2py/f2pyarg.py @@ -544,45 +544,6 @@ def __call__(self, parser, namespace, values, option_string=None): type=pathlib.Path, nargs=argparse.REMAINDER) -################## -# Parser Actions # -################## - - -def callcrackfortran(files, options): - rules.options = options - crackfortran.debug = options['debug'] - crackfortran.verbose = options['verbose'] - if 'module' in options: - crackfortran.f77modulename = options['module'] - if 'skipfuncs' in options: - crackfortran.skipfuncs = options['skipfuncs'] - if 'onlyfuncs' in options: - crackfortran.onlyfuncs = options['onlyfuncs'] - crackfortran.include_paths[:] = options['include_paths'] - crackfortran.dolowercase = options['do-lower'] - postlist = crackfortran.crackfortran(files) - if 'signsfile' in options: - outmess('Saving signatures to file "%s"\n' % (options['signsfile'])) - pyf = crackfortran.crack2fortran(postlist) - if options['signsfile'][-6:] == 'stdout': - sys.stdout.write(pyf) - else: - with open(options['signsfile'], 'w') as f: - f.write(pyf) - if options["coutput"] is None: - for mod in postlist: - mod["coutput"] = "%smodule.c" % mod["name"] - else: - for mod in postlist: - mod["coutput"] = options["coutput"] - if options["f2py_wrapper_output"] is None: - for mod in postlist: - mod["f2py_wrapper_output"] = "%s-f2pywrappers.f" % mod["name"] - else: - for mod in postlist: - mod["f2py_wrapper_output"] = options["f2py_wrapper_output"] - return postlist ################ # Main Process # diff --git a/numpy/f2py/service.py b/numpy/f2py/service.py index fe05bc8a3994..c46450ce272c 100644 --- a/numpy/f2py/service.py +++ b/numpy/f2py/service.py @@ -106,6 +106,119 @@ def check_npfcomp(opt: str): +def _set_options(settings): + crackfortran.reset_global_f2py_vars() + capi_maps.load_f2cmap_file(settings['f2cmap']) + auxfuncs.options = {'verbose': settings['verbose']} + auxfuncs.debugoptions = settings["debug"] + auxfuncs.wrapfuncs = settings['wrapfuncs'] + rules.options = { + 'buildpath': settings['buildpath'], + 'dorestdoc': settings['dorestdoc'], + 'dolatexdoc': settings['dolatexdoc'], + 'shortlatex': settings['shortlatex'], + } + + +def _dict_append(d_out, d_in): + for (k, v) in d_in.items(): + if k not in d_out: + d_out[k] = [] + if isinstance(v, list): + d_out[k] = d_out[k] + v + else: + d_out[k].append(v) + + +def _buildmodules(lst): + cfuncs.buildcfuncs() + outmess('Building modules...\n') + modules, mnames, isusedby = [], [], {} + for item in lst: + if '__user__' in item['name']: + cb_rules.buildcallbacks(item) + else: + if 'use' in item: + for u in item['use'].keys(): + if u not in isusedby: + isusedby[u] = [] + isusedby[u].append(item['name']) + modules.append(item) + mnames.append(item['name']) + ret = {} + for module, name in zip(modules, mnames): + if name in isusedby: + outmess('\tSkipping module "%s" which is used by %s.\n' % ( + name, ','.join('"%s"' % s for s in isusedby[name]))) + else: + um = [] + if 'use' in module: + for u in module['use'].keys(): + if u in isusedby and u in mnames: + um.append(modules[mnames.index(u)]) + else: + outmess( + f'\tModule "{name}" uses nonexisting "{u}" ' + 'which will be ignored.\n') + ret[name] = {} + _dict_append(ret[name], rules.buildmodule(module, um)) + return ret + + +def _generate_signature(postlist, sign_file: str): + outmess(f"Saving signatures to file {sign_file}" + "\n") + pyf = crackfortran.crack2fortran(postlist) + if sign_file in ["-", "stdout"]: + sys.stdout.write(pyf) + else: + with open(sign_file, "w") as f: + f.write(pyf) + +def _parse_postlist(postlist, sign_file: str, verbose: bool): + isusedby = {} + for plist in postlist: + if 'use' in plist: + for u in plist['use'].keys(): + if u not in isusedby: + isusedby[u] = [] + isusedby[u].append(plist['name']) + for plist in postlist: + if plist['block'] == 'python module' and '__user__' in plist['name']: + if plist['name'] in isusedby: + # if not quiet: + outmess( + f'Skipping Makefile build for module "{plist["name"]}" ' + 'which is used by {}\n'.format( + ','.join(f'"{s}"' for s in isusedby[plist['name']]))) + if(sign_file): + if verbose: + outmess( + 'Stopping. Edit the signature file and then run f2py on the signature file: ') + outmess('%s %s\n' % + (os.path.basename(sys.argv[0]), sign_file)) + return + for plist in postlist: + if plist['block'] != 'python module': + # if 'python module' not in options: + outmess( + 'Tip: If your original code is Fortran source then you must use -m option.\n', verbose) + # raise TypeError('All blocks must be python module blocks but got %s' % ( + # repr(postlist[i]['block']))) + +def _callcrackfortran(files: List[Path], module_name: str, include_paths: List[Path], options: Dict[str, Any]): + crackfortran.f77modulename = module_name + crackfortran.include_paths[:] = include_paths + crackfortran.debug = options["debug"] + crackfortran.verbose = options["verbose"] + crackfortran.skipfuncs = options["skipfuncs"] + crackfortran.onlyfuncs = options["onlyfuncs"] + crackfortran.dolowercase = options["do-lower"] + postlist = crackfortran.crackfortran([str(file) for file in files]) + for mod in postlist: + mod["coutput"] = f"{module_name}module.c" + mod["f2py_wrapper_output"] = f"{module_name}-f2pywrappers.f" + return postlist + def segregate_files(files: List[Path]) -> Tuple[List[Path], List[Path], List[Path], List[Path]]: """ Segregate files into three groups: From 4180fcae262d33da2a1cde5ec1d44706d9ab55e4 Mon Sep 17 00:00:00 2001 From: Namami Shanker Date: Sat, 2 Jul 2022 18:45:45 +0530 Subject: [PATCH 019/132] ENH: Initial f2pyarg main arg process function --- numpy/f2py/f2pyarg.py | 43 ++++++++++++++++++++++++++++++++----------- numpy/f2py/service.py | 17 +++++++++++++++++ 2 files changed, 49 insertions(+), 11 deletions(-) diff --git a/numpy/f2py/f2pyarg.py b/numpy/f2py/f2pyarg.py index 0b2388858bd9..67298fe9d94f 100644 --- a/numpy/f2py/f2pyarg.py +++ b/numpy/f2py/f2pyarg.py @@ -552,19 +552,40 @@ def __call__(self, parser, namespace, values, option_string=None): def process_args(args): build_dir = args.build_dir[0] if args.build_dir else tempfile.mkdtemp() + sign_file = None + if(args.hint_signature): + if(args.build_dir): + sign_file = args.build_dir[0] / args.hint_signature[0] + else: + sign_file = args.hint_signature[0] + + settings = { + 'f2cmap': args.f2cmap, + 'verbose': args.verbose, + 'buildpath': build_dir, + 'dorestdoc': args.rest_doc, + 'dolatexdoc': args.latex_doc, + 'shortlatex': args.short_latex, + 'debug': args.debug_capi, + 'wrapfuncs': args.wrap_functions, + } + + file_gen_options = { + 'verbose': args.verbose, + 'module': args.module[0], + 'skipfuncs': getattr(args, 'Skip Functions', []), + 'onlyfuncs': getattr(args, 'Keep Functions', []), + 'include_paths': args.include_paths, + 'do-lower': args.lower, + 'debug': args.debug_capi, + 'wrapfuncs': args.wrap_functions, + } + if args.help: parser.print_help() - elif getattr(args, "Fortran Files"): - print("BOOM") - if args.c: - if args.fcompiler: - print(f"got {args.fcompiler}") - elif args.compiler: - print(args.compiler) - else: - print("Compilation requested without options, using defaults") - else: - parser.print_usage() + if getattr(args, "Fortran Files"): + f77_files, f90_files, out_files, other_files = segregate_files(getattr(args, "Fortran Files")) + generate_files(f77_files + f90_files, args.module[0], args.include_paths, sign_file, file_gen_options, settings) def main(): diff --git a/numpy/f2py/service.py b/numpy/f2py/service.py index c46450ce272c..17c8c33172ac 100644 --- a/numpy/f2py/service.py +++ b/numpy/f2py/service.py @@ -8,6 +8,13 @@ from argparse import Namespace from . import crackfortran +from . import capi_maps +from . import rules +from . import auxfuncs +from . import cfuncs +from . import cb_rules + +outmess = auxfuncs.outmess logger = logging.getLogger("f2py_cli") logger.setLevel(logging.WARNING) @@ -219,6 +226,16 @@ def _callcrackfortran(files: List[Path], module_name: str, include_paths: List[P mod["f2py_wrapper_output"] = f"{module_name}-f2pywrappers.f" return postlist +def generate_files(files: List[Path], module_name: str, include_paths: List[Path], sign_file: str, file_gen_options: Dict[str, Any], settings: Dict[str, Any]): + _set_options(settings) + postlist = _callcrackfortran(files, module_name, include_paths, file_gen_options) + _parse_postlist(postlist, sign_file, file_gen_options["verbose"]) + if(sign_file): + _generate_signature(postlist, sign_file) + if(module_name != 'untitled'): + _buildmodules(postlist) + + def segregate_files(files: List[Path]) -> Tuple[List[Path], List[Path], List[Path], List[Path]]: """ Segregate files into three groups: From a0eab32cc58118eb281078bb72d336223610d244 Mon Sep 17 00:00:00 2001 From: Namami Shanker Date: Sun, 3 Jul 2022 13:43:49 +0530 Subject: [PATCH 020/132] FIX: fix imports --- numpy/f2py/f2pyarg.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/numpy/f2py/f2pyarg.py b/numpy/f2py/f2pyarg.py index 67298fe9d94f..de0f72c364fe 100644 --- a/numpy/f2py/f2pyarg.py +++ b/numpy/f2py/f2pyarg.py @@ -25,7 +25,7 @@ # F2PY imports from . import crackfortran from . import rules -from .service import check_dccomp, check_npfcomp, check_fortran, check_dir +from .service import check_dccomp, check_npfcomp, check_fortran, check_dir, generate_files, segregate_files ################## # Temp Variables # From 528dc4ff2557efef73008c85f6aff8a4c5ecb8f4 Mon Sep 17 00:00:00 2001 From: Namami Shanker Date: Sun, 3 Jul 2022 13:52:22 +0530 Subject: [PATCH 021/132] FIX: Set default verbosity to true like f2py2e --- numpy/f2py/f2pyarg.py | 1 + 1 file changed, 1 insertion(+) diff --git a/numpy/f2py/f2pyarg.py b/numpy/f2py/f2pyarg.py index de0f72c364fe..f9ff79478b54 100644 --- a/numpy/f2py/f2pyarg.py +++ b/numpy/f2py/f2pyarg.py @@ -331,6 +331,7 @@ def __call__(self, parser, namespace, values, option_string=None): parser.add_argument( "--verbose", action="store_true", + default=True, help="Run with extra verbosity.", ) From d2a72b1f97f3d21097d1b1dad07373ea308aaa09 Mon Sep 17 00:00:00 2001 From: Namami Shanker Date: Sun, 3 Jul 2022 15:24:08 +0530 Subject: [PATCH 022/132] MAINT: Remove unneccessary imports --- numpy/f2py/f2pyarg.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/numpy/f2py/f2pyarg.py b/numpy/f2py/f2pyarg.py index f9ff79478b54..8a133bc45f63 100644 --- a/numpy/f2py/f2pyarg.py +++ b/numpy/f2py/f2pyarg.py @@ -22,9 +22,6 @@ from numpy.version import version as __version__ -# F2PY imports -from . import crackfortran -from . import rules from .service import check_dccomp, check_npfcomp, check_fortran, check_dir, generate_files, segregate_files ################## From 475f4df3e60a8eb5ffcc355cdd543dafbec71a04 Mon Sep 17 00:00:00 2001 From: Namami Shanker Date: Sun, 3 Jul 2022 15:24:38 +0530 Subject: [PATCH 023/132] ENH: Handle overwrite-signature flag --- numpy/f2py/f2pyarg.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/numpy/f2py/f2pyarg.py b/numpy/f2py/f2pyarg.py index 8a133bc45f63..21c7b4cac396 100644 --- a/numpy/f2py/f2pyarg.py +++ b/numpy/f2py/f2pyarg.py @@ -15,6 +15,7 @@ implementation in terms of subparsers """ +import os import argparse import logging import pathlib @@ -556,6 +557,8 @@ def process_args(args): sign_file = args.build_dir[0] / args.hint_signature[0] else: sign_file = args.hint_signature[0] + if sign_file and os.path.isfile(sign_file) and not args.overwrite_signature: + print(f'Signature file "{sign_file}" exists!!! Use --overwrite-signature to overwrite.') settings = { 'f2cmap': args.f2cmap, From aa3d300927a15e17d43796c9b75ea676925882e4 Mon Sep 17 00:00:00 2001 From: Namami Shanker Date: Mon, 4 Jul 2022 16:27:04 +0530 Subject: [PATCH 024/132] ENH: update auxfuncs --- numpy/f2py/auxfuncs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/numpy/f2py/auxfuncs.py b/numpy/f2py/auxfuncs.py index 3f9b0ceafa21..de29a42094b4 100644 --- a/numpy/f2py/auxfuncs.py +++ b/numpy/f2py/auxfuncs.py @@ -68,7 +68,7 @@ def outmess(t): def debugcapi(var): - return 'capi' in debugoptions + return debugoptions def _ischaracter(var): From 885c7727c123d24effbc7893d3e1c4345c05cf3b Mon Sep 17 00:00:00 2001 From: Namami Shanker Date: Mon, 4 Jul 2022 17:02:13 +0530 Subject: [PATCH 025/132] ENH: Segregate pyf files --- numpy/f2py/f2pyarg.py | 2 +- numpy/f2py/service.py | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/numpy/f2py/f2pyarg.py b/numpy/f2py/f2pyarg.py index 21c7b4cac396..eda88d2c539b 100644 --- a/numpy/f2py/f2pyarg.py +++ b/numpy/f2py/f2pyarg.py @@ -585,7 +585,7 @@ def process_args(args): if args.help: parser.print_help() if getattr(args, "Fortran Files"): - f77_files, f90_files, out_files, other_files = segregate_files(getattr(args, "Fortran Files")) + f77_files, f90_files, fpy_files, out_files, other_files = segregate_files(getattr(args, "Fortran Files")) generate_files(f77_files + f90_files, args.module[0], args.include_paths, sign_file, file_gen_options, settings) diff --git a/numpy/f2py/service.py b/numpy/f2py/service.py index 17c8c33172ac..f6b7adcced0e 100644 --- a/numpy/f2py/service.py +++ b/numpy/f2py/service.py @@ -245,11 +245,13 @@ def segregate_files(files: List[Path]) -> Tuple[List[Path], List[Path], List[Pat """ f77_ext = ('.f', '.for', '.ftn', '.f77') f90_ext = ('.f90', '.f95', '.f03', '.f08') + pyf_ext = ('.pyf',) out_ext = ('.o', '.out', '.so', '.a') f77_files = [] f90_files = [] out_files = [] + pyf_files = [] other_files = [] for f in files: @@ -260,7 +262,9 @@ def segregate_files(files: List[Path]) -> Tuple[List[Path], List[Path], List[Pat f90_files.append(f) elif ext in out_ext: out_files.append(f) + elif ext in pyf_ext: + pyf_files.append(f) else: other_files.append(f) - return f77_files, f90_files, out_files, other_files \ No newline at end of file + return f77_files, f90_files, pyf_files, out_files, other_files \ No newline at end of file From f39eb6d01dc3ab9aa3bd5557d3bcd550506ab5e4 Mon Sep 17 00:00:00 2001 From: Namami Shanker Date: Mon, 4 Jul 2022 17:56:30 +0530 Subject: [PATCH 026/132] MAINT: Refactoring --- numpy/f2py/f2pyarg.py | 3 ++- numpy/f2py/service.py | 15 +++++++-------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/numpy/f2py/f2pyarg.py b/numpy/f2py/f2pyarg.py index eda88d2c539b..467ff6734f3e 100644 --- a/numpy/f2py/f2pyarg.py +++ b/numpy/f2py/f2pyarg.py @@ -20,6 +20,7 @@ import logging import pathlib import tempfile +from numpy.f2py.auxfuncs import outmess from numpy.version import version as __version__ @@ -80,7 +81,7 @@ def __call__(self, parser, namespace, values, option_string=None): Essentially this returns the semantically valid operation implied by --no """ - setattr(namespace, self.dest, False if option_string.contains("no") else True) + setattr(namespace, self.dest, not option_string.contains("no")) # TODO: Generalize or kill this np.distutils specific helper action class diff --git a/numpy/f2py/service.py b/numpy/f2py/service.py index f6b7adcced0e..5c927b28b1e4 100644 --- a/numpy/f2py/service.py +++ b/numpy/f2py/service.py @@ -175,7 +175,7 @@ def _buildmodules(lst): def _generate_signature(postlist, sign_file: str): outmess(f"Saving signatures to file {sign_file}" + "\n") pyf = crackfortran.crack2fortran(postlist) - if sign_file in ["-", "stdout"]: + if sign_file in {"-", "stdout"}: sys.stdout.write(pyf) else: with open(sign_file, "w") as f: @@ -190,13 +190,12 @@ def _parse_postlist(postlist, sign_file: str, verbose: bool): isusedby[u] = [] isusedby[u].append(plist['name']) for plist in postlist: - if plist['block'] == 'python module' and '__user__' in plist['name']: - if plist['name'] in isusedby: - # if not quiet: - outmess( - f'Skipping Makefile build for module "{plist["name"]}" ' - 'which is used by {}\n'.format( - ','.join(f'"{s}"' for s in isusedby[plist['name']]))) + if plist['block'] == 'python module' and '__user__' in plist['name'] and plist['name'] in isusedby: + # if not quiet: + outmess( + f'Skipping Makefile build for module "{plist["name"]}" ' + 'which is used by {}\n'.format( + ','.join(f'"{s}"' for s in isusedby[plist['name']]))) if(sign_file): if verbose: outmess( From 1df8ee2476fb1a0266738db05c15e1843a31f47e Mon Sep 17 00:00:00 2001 From: Namami Shanker Date: Mon, 4 Jul 2022 17:57:36 +0530 Subject: [PATCH 027/132] MAINT: Rename _parse_postlist to _check_postlist --- numpy/f2py/service.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/numpy/f2py/service.py b/numpy/f2py/service.py index 5c927b28b1e4..63832589e148 100644 --- a/numpy/f2py/service.py +++ b/numpy/f2py/service.py @@ -181,7 +181,7 @@ def _generate_signature(postlist, sign_file: str): with open(sign_file, "w") as f: f.write(pyf) -def _parse_postlist(postlist, sign_file: str, verbose: bool): +def _check_postlist(postlist, sign_file: str, verbose: bool): isusedby = {} for plist in postlist: if 'use' in plist: @@ -228,7 +228,7 @@ def _callcrackfortran(files: List[Path], module_name: str, include_paths: List[P def generate_files(files: List[Path], module_name: str, include_paths: List[Path], sign_file: str, file_gen_options: Dict[str, Any], settings: Dict[str, Any]): _set_options(settings) postlist = _callcrackfortran(files, module_name, include_paths, file_gen_options) - _parse_postlist(postlist, sign_file, file_gen_options["verbose"]) + _check_postlist(postlist, sign_file, file_gen_options["verbose"]) if(sign_file): _generate_signature(postlist, sign_file) if(module_name != 'untitled'): From 15be187957cb4c1e7d15f47f9622aaedae1a18b5 Mon Sep 17 00:00:00 2001 From: Namami Shanker Date: Tue, 5 Jul 2022 17:51:53 +0530 Subject: [PATCH 028/132] Revert "ENH: update auxfuncs" This reverts commit b334020a6b4d57831b7963826abf731fef6d3d5e. --- numpy/f2py/auxfuncs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/numpy/f2py/auxfuncs.py b/numpy/f2py/auxfuncs.py index de29a42094b4..3f9b0ceafa21 100644 --- a/numpy/f2py/auxfuncs.py +++ b/numpy/f2py/auxfuncs.py @@ -68,7 +68,7 @@ def outmess(t): def debugcapi(var): - return debugoptions + return 'capi' in debugoptions def _ischaracter(var): From e13faa1fb98883fff13c539c3541395cc9ba6632 Mon Sep 17 00:00:00 2001 From: Namami Shanker Date: Tue, 5 Jul 2022 18:51:02 +0530 Subject: [PATCH 029/132] FIX: Accept array of debug options rather that boolean flag --- numpy/f2py/f2pyarg.py | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/numpy/f2py/f2pyarg.py b/numpy/f2py/f2pyarg.py index 467ff6734f3e..063334de4c1c 100644 --- a/numpy/f2py/f2pyarg.py +++ b/numpy/f2py/f2pyarg.py @@ -50,6 +50,8 @@ 'gtk+-x11-2.0', 'gtkp_2', 'gtk+-2.0', 'xft', 'freetype2', 'umfpack', 'amd'] +debug_api = ['capi'] + # TODO: Compatibility helper, kill later # From 3.9 onwards should be argparse.BooleanOptionalAction @@ -113,6 +115,23 @@ def __call__(self, parser, namespace, values, option_string=None): else: raise RuntimeError(f"{outvar} is not in {npd_link}") +class DebugLinkHelper(argparse.Action): + """A custom action to work with f2py's --debug-blah""" + + def __call__(self, parser, namespace, values, option_string=None): + """The storage action + + Essentially, split the value on -, store in dest + + """ + items = getattr(namespace, self.dest) or [] + outvar = option_string.split("--debug-")[1] + if outvar in debug_api: + items.append(outvar) + setattr(namespace, self.dest, items) + else: + raise RuntimeError(f"{outvar} is not in {debug_api}") + ########## # Parser # @@ -270,7 +289,10 @@ def __call__(self, parser, namespace, values, option_string=None): parser.add_argument( "--debug-capi", - action="store_true", + dest="debug_api", + default=[], + nargs="*", + action=DebugLinkHelper, help="""Create C/API code that reports the state of the wrappers during runtime. Useful for debugging.""", ) From 28e5bebaea0e8f0bf87cdaccb76e96fbc46fab0c Mon Sep 17 00:00:00 2001 From: Namami Shanker Date: Tue, 5 Jul 2022 18:52:27 +0530 Subject: [PATCH 030/132] FIX: Default module name and default build directories --- numpy/f2py/f2pyarg.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/numpy/f2py/f2pyarg.py b/numpy/f2py/f2pyarg.py index 063334de4c1c..f256e3f66838 100644 --- a/numpy/f2py/f2pyarg.py +++ b/numpy/f2py/f2pyarg.py @@ -220,7 +220,6 @@ def __call__(self, parser, namespace, values, option_string=None): metavar="", type=str, nargs=1, - default="untitled", help="""Name of the module; f2py generates a Python/C API file module.c or extension module . Default is 'untitled'.""", @@ -573,15 +572,19 @@ def __call__(self, parser, namespace, values, option_string=None): def process_args(args): - build_dir = args.build_dir[0] if args.build_dir else tempfile.mkdtemp() + # Default build dir for wrappers is the current directory + # while for compilations its a temporary directory + wrapper_build_dir = args.build_dir[0] if args.build_dir else pathlib.Path.cwd() + compile_build_dir = args.build_dir[0] if args.build_dir else tempfile.mkdtemp() + + # Wrapper generation code sign_file = None if(args.hint_signature): - if(args.build_dir): - sign_file = args.build_dir[0] / args.hint_signature[0] - else: - sign_file = args.hint_signature[0] + sign_file = wrapper_build_dir / args.hint_signature[0] if sign_file and os.path.isfile(sign_file) and not args.overwrite_signature: print(f'Signature file "{sign_file}" exists!!! Use --overwrite-signature to overwrite.') + + module_name = args.module[0] if args.module else None settings = { 'f2cmap': args.f2cmap, From 810aad7012382aea806880fc11f07f2bf90161ea Mon Sep 17 00:00:00 2001 From: Namami Shanker Date: Tue, 5 Jul 2022 18:53:17 +0530 Subject: [PATCH 031/132] MAINT: Fix typo and send pyf files to generate_files --- numpy/f2py/f2pyarg.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/numpy/f2py/f2pyarg.py b/numpy/f2py/f2pyarg.py index f256e3f66838..9ddfb2f276af 100644 --- a/numpy/f2py/f2pyarg.py +++ b/numpy/f2py/f2pyarg.py @@ -611,8 +611,8 @@ def process_args(args): if args.help: parser.print_help() if getattr(args, "Fortran Files"): - f77_files, f90_files, fpy_files, out_files, other_files = segregate_files(getattr(args, "Fortran Files")) - generate_files(f77_files + f90_files, args.module[0], args.include_paths, sign_file, file_gen_options, settings) + f77_files, f90_files, pyf_files, out_files, other_files = segregate_files(getattr(args, "Fortran Files")) + generate_files(pyf_files + f77_files + f90_files, module_name, sign_file, file_gen_options, settings) def main(): From bc8dfa947e457683413f0243bf3638a789a88b1e Mon Sep 17 00:00:00 2001 From: Namami Shanker Date: Tue, 5 Jul 2022 18:55:41 +0530 Subject: [PATCH 032/132] ENH: Send all required options to rules file --- numpy/f2py/f2pyarg.py | 10 ++++++++-- numpy/f2py/service.py | 7 +++++++ 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/numpy/f2py/f2pyarg.py b/numpy/f2py/f2pyarg.py index 9ddfb2f276af..4f41f2f4d4b6 100644 --- a/numpy/f2py/f2pyarg.py +++ b/numpy/f2py/f2pyarg.py @@ -587,14 +587,20 @@ def process_args(args): module_name = args.module[0] if args.module else None settings = { + 'buildpath': wrapper_build_dir, 'f2cmap': args.f2cmap, 'verbose': args.verbose, - 'buildpath': build_dir, 'dorestdoc': args.rest_doc, 'dolatexdoc': args.latex_doc, 'shortlatex': args.short_latex, - 'debug': args.debug_capi, + 'debug': args.debug_api, 'wrapfuncs': args.wrap_functions, + 'do-lower': args.lower, + 'include_paths': args.include_paths, + # Disabing these options from frontend + 'emptygen': True, + 'f2py_wrapper_output': None, + 'coutput': None, } file_gen_options = { diff --git a/numpy/f2py/service.py b/numpy/f2py/service.py index 63832589e148..ed1deefca315 100644 --- a/numpy/f2py/service.py +++ b/numpy/f2py/service.py @@ -124,6 +124,13 @@ def _set_options(settings): 'dorestdoc': settings['dorestdoc'], 'dolatexdoc': settings['dolatexdoc'], 'shortlatex': settings['shortlatex'], + 'coutput': settings['coutput'], + 'f2py_wrapper_output': settings['f2py_wrapper_output'], + 'emptygen': settings['emptygen'], + 'verbose': settings['verbose'], + 'do-lower': settings['do-lower'], + 'f2cmap_file': settings['f2cmap'], + 'include_paths': settings['include_paths'], } From fc68223ea3778e09f97b008eb329e6179fb58f8b Mon Sep 17 00:00:00 2001 From: Namami Shanker Date: Tue, 5 Jul 2022 18:56:58 +0530 Subject: [PATCH 033/132] MAINT: Refactoring service --- numpy/f2py/f2pyarg.py | 5 +++-- numpy/f2py/service.py | 16 +++++++++------- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/numpy/f2py/f2pyarg.py b/numpy/f2py/f2pyarg.py index 4f41f2f4d4b6..2cbb2afbe068 100644 --- a/numpy/f2py/f2pyarg.py +++ b/numpy/f2py/f2pyarg.py @@ -526,6 +526,7 @@ def __call__(self, parser, namespace, values, option_string=None): # TODO: Kill this ASAP # Also collect in to REMAINDER and extract from there +# Flag not working. To be debugged. build_helpers.add_argument( '--link-atlas', '--link-atlas_threads', '--link-atlas_blas', '--link-atlas_blas_threads', '--link-lapack_atlas', @@ -605,12 +606,12 @@ def process_args(args): file_gen_options = { 'verbose': args.verbose, - 'module': args.module[0], + 'module': module_name, 'skipfuncs': getattr(args, 'Skip Functions', []), 'onlyfuncs': getattr(args, 'Keep Functions', []), 'include_paths': args.include_paths, 'do-lower': args.lower, - 'debug': args.debug_capi, + 'debug': args.debug_api, 'wrapfuncs': args.wrap_functions, } diff --git a/numpy/f2py/service.py b/numpy/f2py/service.py index ed1deefca315..666d6ca0e5db 100644 --- a/numpy/f2py/service.py +++ b/numpy/f2py/service.py @@ -113,7 +113,7 @@ def check_npfcomp(opt: str): -def _set_options(settings): +def _set_options(module_name: str, settings: Dict[str, Any]): crackfortran.reset_global_f2py_vars() capi_maps.load_f2cmap_file(settings['f2cmap']) auxfuncs.options = {'verbose': settings['verbose']} @@ -131,6 +131,7 @@ def _set_options(settings): 'do-lower': settings['do-lower'], 'f2cmap_file': settings['f2cmap'], 'include_paths': settings['include_paths'], + 'module': module_name, } @@ -218,9 +219,9 @@ def _check_postlist(postlist, sign_file: str, verbose: bool): # raise TypeError('All blocks must be python module blocks but got %s' % ( # repr(postlist[i]['block']))) -def _callcrackfortran(files: List[Path], module_name: str, include_paths: List[Path], options: Dict[str, Any]): +def _callcrackfortran(files: List[Path], module_name: str, options: Dict[str, Any]): crackfortran.f77modulename = module_name - crackfortran.include_paths[:] = include_paths + crackfortran.include_paths[:] = options['include_paths'] crackfortran.debug = options["debug"] crackfortran.verbose = options["verbose"] crackfortran.skipfuncs = options["skipfuncs"] @@ -228,17 +229,18 @@ def _callcrackfortran(files: List[Path], module_name: str, include_paths: List[P crackfortran.dolowercase = options["do-lower"] postlist = crackfortran.crackfortran([str(file) for file in files]) for mod in postlist: + module_name = module_name or 'untitled' mod["coutput"] = f"{module_name}module.c" mod["f2py_wrapper_output"] = f"{module_name}-f2pywrappers.f" return postlist -def generate_files(files: List[Path], module_name: str, include_paths: List[Path], sign_file: str, file_gen_options: Dict[str, Any], settings: Dict[str, Any]): - _set_options(settings) - postlist = _callcrackfortran(files, module_name, include_paths, file_gen_options) +def generate_files(files: List[Path], module_name: str, sign_file: str, file_gen_options: Dict[str, Any], settings: Dict[str, Any]): + _set_options(module_name, settings) + postlist = _callcrackfortran(files, module_name, file_gen_options) _check_postlist(postlist, sign_file, file_gen_options["verbose"]) if(sign_file): _generate_signature(postlist, sign_file) - if(module_name != 'untitled'): + if(module_name): _buildmodules(postlist) From be8a4b06159ba65286c772ec60a39b164b8f1799 Mon Sep 17 00:00:00 2001 From: Namami Shanker Date: Fri, 8 Jul 2022 13:50:35 +0530 Subject: [PATCH 034/132] MAINT: Refactor for -c flag --- numpy/f2py/f2pyarg.py | 101 +++++++++++++++++++++++++----------------- 1 file changed, 60 insertions(+), 41 deletions(-) diff --git a/numpy/f2py/f2pyarg.py b/numpy/f2py/f2pyarg.py index 2cbb2afbe068..7b93e5a706dd 100644 --- a/numpy/f2py/f2pyarg.py +++ b/numpy/f2py/f2pyarg.py @@ -572,55 +572,74 @@ def __call__(self, parser, namespace, values, option_string=None): ################ -def process_args(args): - # Default build dir for wrappers is the current directory - # while for compilations its a temporary directory - wrapper_build_dir = args.build_dir[0] if args.build_dir else pathlib.Path.cwd() - compile_build_dir = args.build_dir[0] if args.build_dir else tempfile.mkdtemp() - - # Wrapper generation code +def get_build_dir(args): + if(args.build_dir is not None): + return args.build_dir[0] + if(args.c): + return tempfile.mkdtemp() + return pathlib.Path.cwd() + +def get_module_name(args, pyf_files): + if(args.module_name is not None): + return args.module_name[0] + if args.c: + for file in pyf_files: + if name := get_f2py_modulename(file): + return name + return "unititled" + return None + +def get_signature_file(args, build_dir): sign_file = None if(args.hint_signature): - sign_file = wrapper_build_dir / args.hint_signature[0] + sign_file = build_dir / args.hint_signature[0] if sign_file and os.path.isfile(sign_file) and not args.overwrite_signature: print(f'Signature file "{sign_file}" exists!!! Use --overwrite-signature to overwrite.') - - module_name = args.module[0] if args.module else None - - settings = { - 'buildpath': wrapper_build_dir, - 'f2cmap': args.f2cmap, - 'verbose': args.verbose, - 'dorestdoc': args.rest_doc, - 'dolatexdoc': args.latex_doc, - 'shortlatex': args.short_latex, - 'debug': args.debug_api, - 'wrapfuncs': args.wrap_functions, - 'do-lower': args.lower, - 'include_paths': args.include_paths, - # Disabing these options from frontend - 'emptygen': True, - 'f2py_wrapper_output': None, - 'coutput': None, - } - - file_gen_options = { - 'verbose': args.verbose, - 'module': module_name, - 'skipfuncs': getattr(args, 'Skip Functions', []), - 'onlyfuncs': getattr(args, 'Keep Functions', []), - 'include_paths': args.include_paths, - 'do-lower': args.lower, - 'debug': args.debug_api, - 'wrapfuncs': args.wrap_functions, - } +def process_args(args): if args.help: parser.print_help() if getattr(args, "Fortran Files"): - f77_files, f90_files, pyf_files, out_files, other_files = segregate_files(getattr(args, "Fortran Files")) - generate_files(pyf_files + f77_files + f90_files, module_name, sign_file, file_gen_options, settings) - + f77_files, f90_files, pyf_files, obj_files, other_files = segregate_files(getattr(args, "Fortran Files")) + + module_name = get_module_name(args, pyf_files) + build_dir = get_build_dir(args) + sign_file = get_signature_file(args, build_dir) + + # Disutils receives all the options and builds the extension. + if(args.c): + sources = pyf_files + f77_files + f90_files + remove_build_dir = not bool(args.build_dir) + else: + settings = { + 'buildpath': build_dir, + 'f2cmap': args.f2cmap, + 'verbose': args.verbose, + 'dorestdoc': args.rest_doc, + 'dolatexdoc': args.latex_doc, + 'shortlatex': args.short_latex, + 'debug': args.debug_api, + 'wrapfuncs': args.wrap_functions, + 'do-lower': args.lower, + 'include_paths': args.include_paths, + # Disabing these options from frontend + 'emptygen': True, + 'f2py_wrapper_output': None, + 'coutput': None, + } + + file_gen_options = { + 'verbose': args.verbose, + 'module': module_name, + 'skipfuncs': getattr(args, 'Skip Functions', []), + 'onlyfuncs': getattr(args, 'Keep Functions', []), + 'include_paths': args.include_paths, + 'do-lower': args.lower, + 'debug': args.debug_api, + 'wrapfuncs': args.wrap_functions, + } + + generate_files(f77_files + f90_files, module_name, sign_file, file_gen_options, settings) def main(): logger = logging.getLogger("f2py_cli") From d7ad3ecc3520367c37a342a255fbf856cceef4ba Mon Sep 17 00:00:00 2001 From: Namami Shanker Date: Fri, 8 Jul 2022 13:51:53 +0530 Subject: [PATCH 035/132] FIX: Change -c flag default val --- numpy/f2py/f2pyarg.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/numpy/f2py/f2pyarg.py b/numpy/f2py/f2pyarg.py index 7b93e5a706dd..1b23e0f39c64 100644 --- a/numpy/f2py/f2pyarg.py +++ b/numpy/f2py/f2pyarg.py @@ -385,7 +385,7 @@ def __call__(self, parser, namespace, values, option_string=None): # as a default build_helpers.add_argument( "-c", - default="numpy.distutils", + default=False, action="store_true", help="Compilation (via NumPy distutils)" ) From 7409f5a237b5ec0fbf3eb670b0d34d0a8c257f96 Mon Sep 17 00:00:00 2001 From: Namami Shanker Date: Fri, 8 Jul 2022 13:53:04 +0530 Subject: [PATCH 036/132] ENH: Remove alternative macro flags for easier handling --- numpy/f2py/f2pyarg.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/numpy/f2py/f2pyarg.py b/numpy/f2py/f2pyarg.py index 1b23e0f39c64..4acf2857127c 100644 --- a/numpy/f2py/f2pyarg.py +++ b/numpy/f2py/f2pyarg.py @@ -488,20 +488,19 @@ def __call__(self, parser, namespace, values, option_string=None): build_helpers.add_argument( "-U", - "--undef-macros", type=str, nargs="*", action="extend", + dest='undef_macros', help="Undefined macros" ) build_helpers.add_argument( "-D", - "--def-macros", type=str, nargs="*", - action="extend", - help="Defined macros" + dest='define_macros', + help="Define macros" ) build_helpers.add_argument( From da01606c7701f92dfa451d3630da0fdf1e1bbb2c Mon Sep 17 00:00:00 2001 From: Namami Shanker Date: Fri, 8 Jul 2022 13:53:49 +0530 Subject: [PATCH 037/132] ENH: Add action class to process macros --- numpy/f2py/f2pyarg.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/numpy/f2py/f2pyarg.py b/numpy/f2py/f2pyarg.py index 4acf2857127c..1287195a2cf6 100644 --- a/numpy/f2py/f2pyarg.py +++ b/numpy/f2py/f2pyarg.py @@ -132,6 +132,23 @@ def __call__(self, parser, namespace, values, option_string=None): else: raise RuntimeError(f"{outvar} is not in {debug_api}") +class ProcessMacros(argparse.Action): + """Process macros in the form of --Dmacro=value and -Dmacro""" + + def __call__(self, parser, namespace, values, option_string=None): + """The storage action + + Essentially, split the value on -D, store in dest + + """ + items = getattr(namespace, self.dest) or [] + outvar = option_string.split("-D")[1] + if('=' in outvar): + items.append((outvar.split("=")[0], outvar.split("=")[1])) + else: + items.append((outvar, None)) + setattr(namespace, self.dest, items) + ########## # Parser # @@ -499,6 +516,7 @@ def __call__(self, parser, namespace, values, option_string=None): "-D", type=str, nargs="*", + action=ProcessMacros, dest='define_macros', help="Define macros" ) From c2f600f010f6c343e4d706a789943699e4efd983 Mon Sep 17 00:00:00 2001 From: Namami Shanker Date: Fri, 8 Jul 2022 13:54:34 +0530 Subject: [PATCH 038/132] FIX: BoolAction string has no contains method error --- numpy/f2py/f2pyarg.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/numpy/f2py/f2pyarg.py b/numpy/f2py/f2pyarg.py index 1287195a2cf6..91bc9dbc94f6 100644 --- a/numpy/f2py/f2pyarg.py +++ b/numpy/f2py/f2pyarg.py @@ -77,13 +77,13 @@ def __init__(self, option_strings, dest, nargs=None, **kwargs): """ super(BoolAction, self).__init__(option_strings, dest, nargs=0, **kwargs) - def __call__(self, parser, namespace, values, option_string=None): + def __call__(self, parser, namespace, values, option_string: str=None): """The logical negation action Essentially this returns the semantically valid operation implied by --no """ - setattr(namespace, self.dest, not option_string.contains("no")) + setattr(namespace, self.dest, "no" not in option_string) # TODO: Generalize or kill this np.distutils specific helper action class From fd587884b89aa32fc74e94a3a9cf433460a9ed6d Mon Sep 17 00:00:00 2001 From: Namami Shanker Date: Fri, 8 Jul 2022 13:55:18 +0530 Subject: [PATCH 039/132] FIX: NPDLinkHelper setattr failure --- numpy/f2py/f2pyarg.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/numpy/f2py/f2pyarg.py b/numpy/f2py/f2pyarg.py index 91bc9dbc94f6..775b34f6286b 100644 --- a/numpy/f2py/f2pyarg.py +++ b/numpy/f2py/f2pyarg.py @@ -111,7 +111,8 @@ def __call__(self, parser, namespace, values, option_string=None): outvar = option_string.split("--link-")[1] if outvar in npd_link: # replicate the extend functionality - items.extend(outvar) + items.append(outvar) + setattr(namespace, self.dest, items) else: raise RuntimeError(f"{outvar} is not in {npd_link}") From 526d37407c238385197c5dab7c019d0f775ac0d6 Mon Sep 17 00:00:00 2001 From: Namami Shanker Date: Fri, 8 Jul 2022 13:56:06 +0530 Subject: [PATCH 040/132] ENH: Add initial -c flag handling --- numpy/f2py/f2pyarg.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/numpy/f2py/f2pyarg.py b/numpy/f2py/f2pyarg.py index 775b34f6286b..aa32196f1cb3 100644 --- a/numpy/f2py/f2pyarg.py +++ b/numpy/f2py/f2pyarg.py @@ -628,6 +628,12 @@ def process_args(args): if(args.c): sources = pyf_files + f77_files + f90_files remove_build_dir = not bool(args.build_dir) + link_files = args.link_resource + include_dirs = args.include_dirs + library_dirs = args.library_path + libraries = args.library_name + define_macros = args.define_macros + undef_macros = args.undef_macros else: settings = { 'buildpath': build_dir, From 70352a375531b8b01b7efec6a86a48b199416e80 Mon Sep 17 00:00:00 2001 From: Namami Shanker Date: Fri, 8 Jul 2022 13:58:41 +0530 Subject: [PATCH 041/132] ENH: Add get_f2py_modulename function from np.distutils --- numpy/f2py/f2pyarg.py | 2 +- numpy/f2py/service.py | 22 +++++++++++++++++++++- 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/numpy/f2py/f2pyarg.py b/numpy/f2py/f2pyarg.py index aa32196f1cb3..24d6d86b6629 100644 --- a/numpy/f2py/f2pyarg.py +++ b/numpy/f2py/f2pyarg.py @@ -24,7 +24,7 @@ from numpy.version import version as __version__ -from .service import check_dccomp, check_npfcomp, check_fortran, check_dir, generate_files, segregate_files +from .service import check_dccomp, check_npfcomp, check_fortran, check_dir, generate_files, segregate_files, compile_dist, get_f2py_modulename ################## # Temp Variables # diff --git a/numpy/f2py/service.py b/numpy/f2py/service.py index 666d6ca0e5db..538e8a294e1d 100644 --- a/numpy/f2py/service.py +++ b/numpy/f2py/service.py @@ -1,7 +1,7 @@ import sys import os import logging -import tempfile +import re from pathlib import Path from typing import Any, Dict, List, Tuple @@ -19,6 +19,11 @@ logger = logging.getLogger("f2py_cli") logger.setLevel(logging.WARNING) +_f2py_module_name_match = re.compile(r'\s*python\s*module\s*(?P[\w_]+)', + re.I).match +_f2py_user_module_name_match = re.compile(r'\s*python\s*module\s*(?P[\w_]*?' + r'__user__[\w_]*)', re.I).match + def check_fortran(fname: str): """Function which checks @@ -234,6 +239,21 @@ def _callcrackfortran(files: List[Path], module_name: str, options: Dict[str, An mod["f2py_wrapper_output"] = f"{module_name}-f2pywrappers.f" return postlist +def get_f2py_modulename(source): + name = None + _f2py_module_name_match = re.compile(r'\s*python\s*module\s*(?P[\w_]+)', + re.I).match + _f2py_user_module_name_match = re.compile(r'\s*python\s*module\s*(?P[\w_]*?' + r'__user__[\w_]*)', re.I).match + with open(source) as f: + for line in f: + if m := _f2py_module_name_match(line): + if _f2py_user_module_name_match(line): # skip *__user__* names + continue + name = m.group('name') + break + return name + def generate_files(files: List[Path], module_name: str, sign_file: str, file_gen_options: Dict[str, Any], settings: Dict[str, Any]): _set_options(module_name, settings) postlist = _callcrackfortran(files, module_name, file_gen_options) From c8fc7495518c5aaaa70589d6ac9472e48810e02a Mon Sep 17 00:00:00 2001 From: Namami Shanker Date: Fri, 8 Jul 2022 14:00:21 +0530 Subject: [PATCH 042/132] FIX: _check_postlist verbosity error --- numpy/f2py/service.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/numpy/f2py/service.py b/numpy/f2py/service.py index 538e8a294e1d..36287e6d9a32 100644 --- a/numpy/f2py/service.py +++ b/numpy/f2py/service.py @@ -220,7 +220,7 @@ def _check_postlist(postlist, sign_file: str, verbose: bool): if plist['block'] != 'python module': # if 'python module' not in options: outmess( - 'Tip: If your original code is Fortran source then you must use -m option.\n', verbose) + 'Tip: If your original code is Fortran source then you must use -m option.\n') # raise TypeError('All blocks must be python module blocks but got %s' % ( # repr(postlist[i]['block']))) From 15b7baa586caad6e84c9d8a96056f668ca12507b Mon Sep 17 00:00:00 2001 From: Namami Shanker Date: Fri, 8 Jul 2022 19:53:45 +0530 Subject: [PATCH 043/132] FIX: Change module_name to module --- numpy/f2py/f2pyarg.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/numpy/f2py/f2pyarg.py b/numpy/f2py/f2pyarg.py index 24d6d86b6629..cb1c1a5a2473 100644 --- a/numpy/f2py/f2pyarg.py +++ b/numpy/f2py/f2pyarg.py @@ -598,8 +598,8 @@ def get_build_dir(args): return pathlib.Path.cwd() def get_module_name(args, pyf_files): - if(args.module_name is not None): - return args.module_name[0] + if(args.module is not None): + return args.module[0] if args.c: for file in pyf_files: if name := get_f2py_modulename(file): From dea01ca29ebff3641f3adf8e999f418398993813 Mon Sep 17 00:00:00 2001 From: Namami Shanker Date: Fri, 8 Jul 2022 20:49:10 +0530 Subject: [PATCH 044/132] FIX: Parse skip and only arguments --- numpy/f2py/f2pyarg.py | 28 +++++++++++++++++++++++----- 1 file changed, 23 insertions(+), 5 deletions(-) diff --git a/numpy/f2py/f2pyarg.py b/numpy/f2py/f2pyarg.py index cb1c1a5a2473..b1cf6d6b5acf 100644 --- a/numpy/f2py/f2pyarg.py +++ b/numpy/f2py/f2pyarg.py @@ -209,7 +209,6 @@ def __call__(self, parser, namespace, values, option_string=None): metavar="", action="extend", # List storage nargs="*", - type=check_fortran, # Returns a pathlib.Path help="""Paths to fortran/signature files that will be scanned for in order to determine their signatures.""", ) @@ -614,11 +613,30 @@ def get_signature_file(args, build_dir): if sign_file and os.path.isfile(sign_file) and not args.overwrite_signature: print(f'Signature file "{sign_file}" exists!!! Use --overwrite-signature to overwrite.') +def segregate_posn_args(args): + # Currently, argparse does not recognise 'skip:' and 'only:' as optional args + # and clubs them all in "Fortran Files" attr. This function segregates them. + funcs = {"skip:": [], "only:": []} + mode = "file" + files = [] + for arg in getattr(args, "Fortran Files"): + if arg in funcs: + mode = arg + elif arg == ':' and mode in funcs: + mode = "file" + elif mode == "file": + files.append(arg) + else: + funcs[mode].append(arg) + return files, funcs['skip:'], funcs['only:'] + def process_args(args): if args.help: parser.print_help() + parser.exit() + files, skip_funcs, only_funcs = segregate_posn_args(args) if getattr(args, "Fortran Files"): - f77_files, f90_files, pyf_files, obj_files, other_files = segregate_files(getattr(args, "Fortran Files")) + f77_files, f90_files, pyf_files, obj_files, other_files = segregate_files(files) module_name = get_module_name(args, pyf_files) build_dir = get_build_dir(args) @@ -653,10 +671,10 @@ def process_args(args): } file_gen_options = { - 'verbose': args.verbose, 'module': module_name, - 'skipfuncs': getattr(args, 'Skip Functions', []), - 'onlyfuncs': getattr(args, 'Keep Functions', []), + 'skipfuncs': skip_funcs, + 'onlyfuncs': only_funcs, + 'verbose': args.verbose, 'include_paths': args.include_paths, 'do-lower': args.lower, 'debug': args.debug_api, From 392101b4de09144e562e17b1911aeb5c412d9257 Mon Sep 17 00:00:00 2001 From: Namami Shanker Date: Fri, 8 Jul 2022 20:53:30 +0530 Subject: [PATCH 045/132] FIX: Segregate and return path of input files --- numpy/f2py/service.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/numpy/f2py/service.py b/numpy/f2py/service.py index 36287e6d9a32..19abfe4dc468 100644 --- a/numpy/f2py/service.py +++ b/numpy/f2py/service.py @@ -264,7 +264,7 @@ def generate_files(files: List[Path], module_name: str, sign_file: str, file_gen _buildmodules(postlist) -def segregate_files(files: List[Path]) -> Tuple[List[Path], List[Path], List[Path], List[Path]]: +def segregate_files(files: List[str]) -> Tuple[List[Path], List[Path], List[Path], List[Path]]: """ Segregate files into three groups: * .f files @@ -283,6 +283,7 @@ def segregate_files(files: List[Path]) -> Tuple[List[Path], List[Path], List[Pat other_files = [] for f in files: + f = Path(f) ext = os.path.splitext(f)[1] if ext in f77_ext: f77_files.append(f) From 50e2c8585f2ae5fe387e49060a6c57f15e73a050 Mon Sep 17 00:00:00 2001 From: Namami Shanker Date: Mon, 11 Jul 2022 20:25:11 +0530 Subject: [PATCH 046/132] FIX: data type of f77flags and f90flags --- numpy/f2py/f2pyarg.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/numpy/f2py/f2pyarg.py b/numpy/f2py/f2pyarg.py index b1cf6d6b5acf..36c5143451b1 100644 --- a/numpy/f2py/f2pyarg.py +++ b/numpy/f2py/f2pyarg.py @@ -444,7 +444,6 @@ def __call__(self, parser, namespace, values, option_string=None): build_helpers.add_argument( "--f77flags", nargs="*", - type=pathlib.Path, action="extend", help="Specify F77 compiler flags" ) @@ -452,7 +451,6 @@ def __call__(self, parser, namespace, values, option_string=None): build_helpers.add_argument( "--f90flags", nargs="*", - type=pathlib.Path, action="extend", help="Specify F90 compiler flags" ) From 9f3efd172992ed0237bf42f1f0e6227aa3d19d4f Mon Sep 17 00:00:00 2001 From: Namami Shanker Date: Mon, 11 Jul 2022 20:25:54 +0530 Subject: [PATCH 047/132] FIX: Return signature file --- numpy/f2py/f2pyarg.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/numpy/f2py/f2pyarg.py b/numpy/f2py/f2pyarg.py index 36c5143451b1..a0332fa3a234 100644 --- a/numpy/f2py/f2pyarg.py +++ b/numpy/f2py/f2pyarg.py @@ -610,6 +610,8 @@ def get_signature_file(args, build_dir): sign_file = build_dir / args.hint_signature[0] if sign_file and os.path.isfile(sign_file) and not args.overwrite_signature: print(f'Signature file "{sign_file}" exists!!! Use --overwrite-signature to overwrite.') + parser.exit() + return sign_file def segregate_posn_args(args): # Currently, argparse does not recognise 'skip:' and 'only:' as optional args From e902559009fe1e39ff6179ecdc3dc251e7740338 Mon Sep 17 00:00:00 2001 From: Namami Shanker Date: Mon, 11 Jul 2022 20:30:36 +0530 Subject: [PATCH 048/132] MAINT: Refactor file segregation --- numpy/f2py/f2pyarg.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/numpy/f2py/f2pyarg.py b/numpy/f2py/f2pyarg.py index a0332fa3a234..b29512dfef94 100644 --- a/numpy/f2py/f2pyarg.py +++ b/numpy/f2py/f2pyarg.py @@ -635,8 +635,7 @@ def process_args(args): parser.print_help() parser.exit() files, skip_funcs, only_funcs = segregate_posn_args(args) - if getattr(args, "Fortran Files"): - f77_files, f90_files, pyf_files, obj_files, other_files = segregate_files(files) + f77_files, f90_files, pyf_files, obj_files, other_files = segregate_files(files) module_name = get_module_name(args, pyf_files) build_dir = get_build_dir(args) From bb41e540d5a1bac7990545ce10ad4ebb74c1d646 Mon Sep 17 00:00:00 2001 From: Namami Shanker Date: Mon, 11 Jul 2022 20:33:40 +0530 Subject: [PATCH 049/132] ENH: Support for -include
flag --- numpy/f2py/f2pyarg.py | 88 ++++++++++++++++++++++--------------------- 1 file changed, 46 insertions(+), 42 deletions(-) diff --git a/numpy/f2py/f2pyarg.py b/numpy/f2py/f2pyarg.py index b29512dfef94..b94c66af009f 100644 --- a/numpy/f2py/f2pyarg.py +++ b/numpy/f2py/f2pyarg.py @@ -586,6 +586,9 @@ def __call__(self, parser, namespace, values, option_string=None): # Main Process # ################ +def get_additional_headers(rem): + return [val[8:] for val in rem if val[:8] == '-include'] + def get_build_dir(args): if(args.build_dir is not None): @@ -630,7 +633,7 @@ def segregate_posn_args(args): funcs[mode].append(arg) return files, funcs['skip:'], funcs['only:'] -def process_args(args): +def process_args(args, rem): if args.help: parser.print_help() parser.exit() @@ -641,53 +644,54 @@ def process_args(args): build_dir = get_build_dir(args) sign_file = get_signature_file(args, build_dir) - # Disutils receives all the options and builds the extension. - if(args.c): - sources = pyf_files + f77_files + f90_files - remove_build_dir = not bool(args.build_dir) - link_files = args.link_resource - include_dirs = args.include_dirs - library_dirs = args.library_path - libraries = args.library_name - define_macros = args.define_macros - undef_macros = args.undef_macros - else: - settings = { - 'buildpath': build_dir, - 'f2cmap': args.f2cmap, - 'verbose': args.verbose, - 'dorestdoc': args.rest_doc, - 'dolatexdoc': args.latex_doc, - 'shortlatex': args.short_latex, - 'debug': args.debug_api, - 'wrapfuncs': args.wrap_functions, - 'do-lower': args.lower, - 'include_paths': args.include_paths, - # Disabing these options from frontend - 'emptygen': True, - 'f2py_wrapper_output': None, - 'coutput': None, - } - - file_gen_options = { - 'module': module_name, - 'skipfuncs': skip_funcs, - 'onlyfuncs': only_funcs, - 'verbose': args.verbose, - 'include_paths': args.include_paths, - 'do-lower': args.lower, - 'debug': args.debug_api, - 'wrapfuncs': args.wrap_functions, - } + headers = get_additional_headers(rem) + # TODO: Refine rules settings. Read codebase and remove unused ones + rules_setts = { + 'module': module_name, + 'buildpath': build_dir, + 'dorestdoc': args.rest_doc, + 'dolatexdoc': args.latex_doc, + 'shortlatex': args.short_latex, + 'verbose': args.verbose, + 'do-lower': args.lower, + 'f2cmap_file': args.f2cmap, + 'include_paths': args.include_paths, + 'coutput': None, + 'f2py_wrapper_output': None, + 'emptygen': True, + } + crackfortran_setts = { + 'module': module_name, + 'skipfuncs': skip_funcs, + 'onlyfuncs': only_funcs, + 'verbose': args.verbose, + 'include_paths': args.include_paths, + 'do-lower': args.lower, + 'debug': args.debug_api, + 'wrapfuncs': args.wrap_functions, + } + capi_maps_setts = { + 'f2cmap': args.f2cmap, + 'headers': headers, + } + auxfuncs_setts = { + 'verbose': args.verbose, + 'debug': args.debug_api, + 'wrapfuncs': args.wrap_functions, + } + + wrapper_settings(rules_setts, crackfortran_setts, capi_maps_setts, auxfuncs_setts) + generate_files(f77_files + f90_files, module_name, sign_file, file_gen_options, settings) def main(): logger = logging.getLogger("f2py_cli") logger.setLevel(logging.WARNING) - args = parser.parse_args() - process_args(args) - + args, rem = parser.parse_known_args() + # since argparse can't handle '-include
' + # we filter it out into rem and parse it manually. + process_args(args, rem) if __name__ == "__main__": main() From 39ef20589fa5cbaf2ef2a53625c4a5bac8742c3c Mon Sep 17 00:00:00 2001 From: Namami Shanker Date: Mon, 11 Jul 2022 20:40:02 +0530 Subject: [PATCH 050/132] MAINT: Refactor wrapper generation settings --- numpy/f2py/service.py | 67 ++++++++++++++++++++++--------------------- 1 file changed, 34 insertions(+), 33 deletions(-) diff --git a/numpy/f2py/service.py b/numpy/f2py/service.py index 19abfe4dc468..f7aca233a12d 100644 --- a/numpy/f2py/service.py +++ b/numpy/f2py/service.py @@ -117,28 +117,32 @@ def check_npfcomp(opt: str): raise RuntimeError(f"{opt} is not an np.distutils supported compiler, choose from {fchoices}") +def _set_additional_headers(headers: List[str]): + for header in headers: + cfuncs.outneeds['userincludes'].append(header[1:-1]) + cfuncs.userincludes[header[1:-1]] = f"#include {header}" -def _set_options(module_name: str, settings: Dict[str, Any]): +def _set_crackfortran(crackfortran_setts): crackfortran.reset_global_f2py_vars() - capi_maps.load_f2cmap_file(settings['f2cmap']) - auxfuncs.options = {'verbose': settings['verbose']} - auxfuncs.debugoptions = settings["debug"] - auxfuncs.wrapfuncs = settings['wrapfuncs'] - rules.options = { - 'buildpath': settings['buildpath'], - 'dorestdoc': settings['dorestdoc'], - 'dolatexdoc': settings['dolatexdoc'], - 'shortlatex': settings['shortlatex'], - 'coutput': settings['coutput'], - 'f2py_wrapper_output': settings['f2py_wrapper_output'], - 'emptygen': settings['emptygen'], - 'verbose': settings['verbose'], - 'do-lower': settings['do-lower'], - 'f2cmap_file': settings['f2cmap'], - 'include_paths': settings['include_paths'], - 'module': module_name, - } - + crackfortran.f77modulename = crackfortran_setts["module"] + crackfortran.include_paths[:] = crackfortran_setts["include_paths"] + crackfortran.debug = crackfortran_setts["debug"] + crackfortran.verbose = crackfortran_setts["verbose"] + crackfortran.skipfuncs = crackfortran_setts["skipfuncs"] + crackfortran.onlyfuncs = crackfortran_setts["onlyfuncs"] + crackfortran.dolowercase = crackfortran_setts["do-lower"] + +def _set_rules(rules_setts): + rules.options = rules_setts + +def _set_capi_maps(capi_maps_setts): + capi_maps.load_f2cmap_file(capi_maps_setts["f2cmap"]) + _set_additional_headers(capi_maps_setts["headers"]) + +def _set_auxfuncs(aux_funcs_setts): + auxfuncs.options = {'verbose': aux_funcs_setts['verbose']} + auxfuncs.debugoptions = aux_funcs_setts["debug"] + auxfuncs.wrapfuncs = aux_funcs_setts['wrapfuncs'] def _dict_append(d_out, d_in): for (k, v) in d_in.items(): @@ -224,17 +228,9 @@ def _check_postlist(postlist, sign_file: str, verbose: bool): # raise TypeError('All blocks must be python module blocks but got %s' % ( # repr(postlist[i]['block']))) -def _callcrackfortran(files: List[Path], module_name: str, options: Dict[str, Any]): - crackfortran.f77modulename = module_name - crackfortran.include_paths[:] = options['include_paths'] - crackfortran.debug = options["debug"] - crackfortran.verbose = options["verbose"] - crackfortran.skipfuncs = options["skipfuncs"] - crackfortran.onlyfuncs = options["onlyfuncs"] - crackfortran.dolowercase = options["do-lower"] +def _callcrackfortran(files: List[Path], module_name: str): postlist = crackfortran.crackfortran([str(file) for file in files]) for mod in postlist: - module_name = module_name or 'untitled' mod["coutput"] = f"{module_name}module.c" mod["f2py_wrapper_output"] = f"{module_name}-f2pywrappers.f" return postlist @@ -254,10 +250,15 @@ def get_f2py_modulename(source): break return name -def generate_files(files: List[Path], module_name: str, sign_file: str, file_gen_options: Dict[str, Any], settings: Dict[str, Any]): - _set_options(module_name, settings) - postlist = _callcrackfortran(files, module_name, file_gen_options) - _check_postlist(postlist, sign_file, file_gen_options["verbose"]) +def wrapper_settings(rules_setts, crackfortran_setts, capi_maps_setts, auxfuncs_setts): + _set_rules(rules_setts) + _set_crackfortran(crackfortran_setts) + _set_capi_maps(capi_maps_setts) + _set_auxfuncs(auxfuncs_setts) + +def generate_files(files: List[Path], module_name: str, sign_file: str): + postlist = _callcrackfortran(files, module_name) + _check_postlist(postlist, sign_file) if(sign_file): _generate_signature(postlist, sign_file) if(module_name): From 5d9f60717d6fa5f155b1e7f713e8ca9576dd5784 Mon Sep 17 00:00:00 2001 From: Namami Shanker Date: Mon, 11 Jul 2022 20:40:43 +0530 Subject: [PATCH 051/132] FIX: Import f2py2e rather than f2py for run_main --- numpy/distutils/command/build_src.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/numpy/distutils/command/build_src.py b/numpy/distutils/command/build_src.py index 5581011f6f22..4047ffcb4fc7 100644 --- a/numpy/distutils/command/build_src.py +++ b/numpy/distutils/command/build_src.py @@ -558,8 +558,8 @@ def f2py_sources(self, sources, extension): and not skip_f2py: log.info("f2py:> %s" % (target_file)) self.mkpath(target_dir) - import numpy.f2py - numpy.f2py.run_main(f2py_options + ['--lower', + from numpy.f2py import f2py2e + f2py2e.run_main(f2py_options + ['--lower', '--build-dir', target_dir]+\ ['-m', ext_name]+f_sources) else: From 9ad8f8183354706f141a7932686490bf1085d639 Mon Sep 17 00:00:00 2001 From: Namami Shanker Date: Mon, 11 Jul 2022 20:42:21 +0530 Subject: [PATCH 052/132] ENH: Generate comp flagsfrom parser namespace --- numpy/f2py/f2pyarg.py | 77 +++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 75 insertions(+), 2 deletions(-) diff --git a/numpy/f2py/f2pyarg.py b/numpy/f2py/f2pyarg.py index b94c66af009f..e4fbfdf3af0b 100644 --- a/numpy/f2py/f2pyarg.py +++ b/numpy/f2py/f2pyarg.py @@ -589,6 +589,62 @@ def __call__(self, parser, namespace, values, option_string=None): def get_additional_headers(rem): return [val[8:] for val in rem if val[:8] == '-include'] +def get_f2pyflags_dist(args, skip_funcs, only_funcs, rem): + # Distutils requires 'f2py_options' which will be a subset of + # sys.argv array received. This function reconstructs the array + # from received args. + f2py_flags = [] + if(args.wrap_functions): + f2py_flags.append('--wrap-functions') + else: + f2py_flags.append('--no-wrap-functions') + if(args.lower): + f2py_flags.append('--lower') + else: + f2py_flags.append('--no-lower') + if(args.debug_api): + f2py_flags.append('--debug-capi') + if(args.quiet): + f2py_flags.append('--quiet') + f2py_flags.append("--skip-empty-wrappers") + if(skip_funcs): + f2py_flags.extend(['skip:']+skip_funcs + [':']) + if(only_funcs): + f2py_flags.extend(['only:']+only_funcs + [':']) + if(args.include_paths): + f2py_flags.extend(['--include-paths']+args.include_paths) + return f2py_flags + +def get_fortran_library_flags(args): + flib_flags = [] + if args.fcompiler: + flib_flags.append(f'--fcompiler={args.fcompiler[0]}') + if args.compiler: + flib_flags.append(f'--compiler={args.compiler[0]}') + return flib_flags + +def get_fortran_compiler_flags(args): + fc_flags = [] + if(args.help_fcompiler): + fc_flags.append('--help-fcompiler') + if(args.f77exec): + fc_flags.append(f'--f77exec={str(args.f77exec[0])}') + if(args.f90exec): + fc_flags.append(f'--f90exec={str(args.f90exec[0])}') + if(args.f77flags): + fc_flags.append(f'--f77flags={" ".join(args.f77flags)}') + if(args.f90flags): + fc_flags.append(f'--f90flags={" ".join(args.f90flags)}') + if(args.arch): + fc_flags.append(f'--arch={" ".join(args.arch)}') + if(args.opt): + fc_flags.append(f'--opt={" ".join(args.opt)}') + if(args.noopt): + fc_flags.append('--noopt') + if(args.noarch): + fc_flags.append('--noarch') + if(args.debug): + fc_flags.append('--debug') def get_build_dir(args): if(args.build_dir is not None): @@ -682,8 +738,25 @@ def process_args(args, rem): wrapper_settings(rules_setts, crackfortran_setts, capi_maps_setts, auxfuncs_setts) - - generate_files(f77_files + f90_files, module_name, sign_file, file_gen_options, settings) + # Disutils receives all the options and builds the extension. + if(args.c): + remove_build_dir = not bool(args.build_dir) + link_resource = args.link_resource + f2py_flags = get_f2pyflags_dist(args, skip_funcs, only_funcs, rem) + fc_flags = get_fortran_compiler_flags(args) + flib_flags = get_fortran_library_flags(args) + + ext_args = { + 'name': module_name, + 'sources': pyf_files + f77_files + f90_files, + 'include_dirs': args.include_dirs, + 'library_dirs': args.library_path, + 'libraries': args.library_name, + 'define_macros': args.define_macros, + 'undef_macros': args.undef_macros, + 'extra_objects': obj_files, + 'f2py_options': f2py_flags, + } def main(): logger = logging.getLogger("f2py_cli") From 02e9ecd5b109da2f645cc2df3e1f4ab14cb57e21 Mon Sep 17 00:00:00 2001 From: Namami Shanker Date: Mon, 11 Jul 2022 20:45:09 +0530 Subject: [PATCH 053/132] ENH: Add f2pyarg distutils compilation --- numpy/f2py/f2pyarg.py | 5 ++++- numpy/f2py/service.py | 33 +++++++++++++++++++++++++++++++++ 2 files changed, 37 insertions(+), 1 deletion(-) diff --git a/numpy/f2py/f2pyarg.py b/numpy/f2py/f2pyarg.py index e4fbfdf3af0b..249e3381bcfc 100644 --- a/numpy/f2py/f2pyarg.py +++ b/numpy/f2py/f2pyarg.py @@ -24,7 +24,7 @@ from numpy.version import version as __version__ -from .service import check_dccomp, check_npfcomp, check_fortran, check_dir, generate_files, segregate_files, compile_dist, get_f2py_modulename +from .service import check_dccomp, check_npfcomp, check_dir, generate_files, segregate_files, get_f2py_modulename, wrapper_settings, compile_dist ################## # Temp Variables # @@ -757,6 +757,9 @@ def process_args(args, rem): 'extra_objects': obj_files, 'f2py_options': f2py_flags, } + compile_dist(ext_args, link_resource, build_dir, fc_flags, flib_flags, args.quiet, remove_build_dir) + else: + generate_files(f77_files + f90_files, module_name, sign_file) def main(): logger = logging.getLogger("f2py_cli") diff --git a/numpy/f2py/service.py b/numpy/f2py/service.py index f7aca233a12d..5d72d30990aa 100644 --- a/numpy/f2py/service.py +++ b/numpy/f2py/service.py @@ -7,6 +7,11 @@ from typing import Any, Dict, List, Tuple from argparse import Namespace +# Distutil dependencies +from numpy.distutils.misc_util import dict_append +from numpy.distutils.system_info import get_info +from numpy.distutils.core import setup, Extension + from . import crackfortran from . import capi_maps from . import rules @@ -14,6 +19,8 @@ from . import cfuncs from . import cb_rules +from .utils import get_f2py_dir + outmess = auxfuncs.outmess logger = logging.getLogger("f2py_cli") @@ -235,6 +242,14 @@ def _callcrackfortran(files: List[Path], module_name: str): mod["f2py_wrapper_output"] = f"{module_name}-f2pywrappers.f" return postlist +def _set_dependencies_dist(ext_args, link_resource): + for dep in link_resource: + info = get_info(dep) + if not info: + outmess('No %s resources found in system' + ' (try `f2py --help-link`)\n' % (repr(n))) + dict_append(ext_args, **link_resource) + def get_f2py_modulename(source): name = None _f2py_module_name_match = re.compile(r'\s*python\s*module\s*(?P[\w_]+)', @@ -264,6 +279,24 @@ def generate_files(files: List[Path], module_name: str, sign_file: str): if(module_name): _buildmodules(postlist) +def compile_dist(ext_args, link_resources, build_dir, fc_flags, flib_flags, quiet_build, remove_build_dir=True): + _set_dependencies_dist(ext_args, link_resources) + f2py_dir = get_f2py_dir() + ext = Extension(**ext_args) + f2py_build_flags = ['--quiet'] if quiet_build else ['--verbose'] + f2py_build_flags.extend( ['build', '--build-temp', str(build_dir), + '--build-base', str(build_dir), + '--build-platlib', '.', + '--disable-optimization']) + if fc_flags: + f2py_build_flags.extend(['config_fc'] + fc_flags) + if flib_flags: + f2py_build_flags.extend(['build_ext'] + flib_flags) + setup(ext_modules=[ext], script_name=f2py_dir, script_args=f2py_build_flags) + if(remove_build_dir and os.path.exists(build_dir)): + import shutil + outmess('Removing build directory %s\n' % (build_dir)) + shutil.rmtree(build_dir) def segregate_files(files: List[str]) -> Tuple[List[Path], List[Path], List[Path], List[Path]]: """ From 19b39484e754f5c54bac3fe0a9b164d98bc4f2f0 Mon Sep 17 00:00:00 2001 From: Namami Shanker Date: Mon, 11 Jul 2022 20:45:42 +0530 Subject: [PATCH 054/132] FIX: link_resource default value --- numpy/f2py/f2pyarg.py | 1 + 1 file changed, 1 insertion(+) diff --git a/numpy/f2py/f2pyarg.py b/numpy/f2py/f2pyarg.py index 249e3381bcfc..24c8a04b7333 100644 --- a/numpy/f2py/f2pyarg.py +++ b/numpy/f2py/f2pyarg.py @@ -569,6 +569,7 @@ def __call__(self, parser, namespace, values, option_string=None): '--link-umfpack', '--link-amd', metavar="--link-", dest="link_resource", + default=[], nargs="*", action=NPDLinkHelper, help="The link helpers for numpy distutils" From f10121185628716f72c1a2fb531d8a1333b71dd7 Mon Sep 17 00:00:00 2001 From: Namami Shanker Date: Mon, 11 Jul 2022 20:46:04 +0530 Subject: [PATCH 055/132] ENH: Add util file for f2py --- numpy/f2py/utils.py | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 numpy/f2py/utils.py diff --git a/numpy/f2py/utils.py b/numpy/f2py/utils.py new file mode 100644 index 000000000000..71298cf047d8 --- /dev/null +++ b/numpy/f2py/utils.py @@ -0,0 +1,7 @@ +"""Global f2py utilities.""" + +import os + +def get_f2py_dir(): + """Return the directory where f2py is installed.""" + return os.path.dirname(os.path.abspath(__file__)) \ No newline at end of file From fc2a70871745f08dc4257c4ead77ace735c2147c Mon Sep 17 00:00:00 2001 From: Namami Shanker Date: Mon, 11 Jul 2022 20:46:43 +0530 Subject: [PATCH 056/132] MAINT: Minor refactoring --- numpy/f2py/service.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/numpy/f2py/service.py b/numpy/f2py/service.py index 5d72d30990aa..110214abcc62 100644 --- a/numpy/f2py/service.py +++ b/numpy/f2py/service.py @@ -205,7 +205,7 @@ def _generate_signature(postlist, sign_file: str): with open(sign_file, "w") as f: f.write(pyf) -def _check_postlist(postlist, sign_file: str, verbose: bool): +def _check_postlist(postlist, sign_file: str): isusedby = {} for plist in postlist: if 'use' in plist: @@ -221,11 +221,10 @@ def _check_postlist(postlist, sign_file: str, verbose: bool): 'which is used by {}\n'.format( ','.join(f'"{s}"' for s in isusedby[plist['name']]))) if(sign_file): - if verbose: - outmess( - 'Stopping. Edit the signature file and then run f2py on the signature file: ') - outmess('%s %s\n' % - (os.path.basename(sys.argv[0]), sign_file)) + outmess( + 'Stopping. Edit the signature file and then run f2py on the signature file: ') + outmess('%s %s\n' % + (os.path.basename(sys.argv[0]), sign_file)) return for plist in postlist: if plist['block'] != 'python module': From 2e38e83889a8bf4432ffb5530badf383002a2c82 Mon Sep 17 00:00:00 2001 From: Namami Shanker Date: Tue, 12 Jul 2022 19:31:20 +0530 Subject: [PATCH 057/132] FIX: Import f2py2e instead of f2py --- numpy/distutils/command/build_src.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/numpy/distutils/command/build_src.py b/numpy/distutils/command/build_src.py index 4047ffcb4fc7..9b42e137543c 100644 --- a/numpy/distutils/command/build_src.py +++ b/numpy/distutils/command/build_src.py @@ -539,8 +539,8 @@ def f2py_sources(self, sources, extension): if (self.force or newer_group(depends, target_file, 'newer')) \ and not skip_f2py: log.info("f2py: %s" % (source)) - import numpy.f2py - numpy.f2py.run_main(f2py_options + from numpy.f2py import f2py2e + f2py2e.run_main(f2py_options + ['--build-dir', target_dir, source]) else: log.debug(" skipping '%s' f2py interface (up-to-date)" % (source)) From bbad3ea423e21f207b9acd9fc5d2cc9455dea0c9 Mon Sep 17 00:00:00 2001 From: Namami Shanker Date: Tue, 12 Jul 2022 19:33:14 +0530 Subject: [PATCH 058/132] MAINT: Remove unused imports --- numpy/f2py/f2pyarg.py | 1 - 1 file changed, 1 deletion(-) diff --git a/numpy/f2py/f2pyarg.py b/numpy/f2py/f2pyarg.py index 24c8a04b7333..5144ed322437 100644 --- a/numpy/f2py/f2pyarg.py +++ b/numpy/f2py/f2pyarg.py @@ -20,7 +20,6 @@ import logging import pathlib import tempfile -from numpy.f2py.auxfuncs import outmess from numpy.version import version as __version__ From 80192f9042b66bac3f1df185426de2eac8c1bf30 Mon Sep 17 00:00:00 2001 From: Namami Shanker Date: Tue, 12 Jul 2022 20:04:32 +0530 Subject: [PATCH 059/132] ENH: Use context manager for build_dir management --- numpy/f2py/f2pyarg.py | 126 +++++++++++++++++++++--------------------- numpy/f2py/service.py | 6 +- numpy/f2py/utils.py | 24 +++++++- 3 files changed, 87 insertions(+), 69 deletions(-) diff --git a/numpy/f2py/f2pyarg.py b/numpy/f2py/f2pyarg.py index 5144ed322437..80c59192ae96 100644 --- a/numpy/f2py/f2pyarg.py +++ b/numpy/f2py/f2pyarg.py @@ -24,6 +24,7 @@ from numpy.version import version as __version__ from .service import check_dccomp, check_npfcomp, check_dir, generate_files, segregate_files, get_f2py_modulename, wrapper_settings, compile_dist +from .utils import open_build_dir ################## # Temp Variables # @@ -696,70 +697,69 @@ def process_args(args, rem): files, skip_funcs, only_funcs = segregate_posn_args(args) f77_files, f90_files, pyf_files, obj_files, other_files = segregate_files(files) - module_name = get_module_name(args, pyf_files) - build_dir = get_build_dir(args) - sign_file = get_signature_file(args, build_dir) - - headers = get_additional_headers(rem) - # TODO: Refine rules settings. Read codebase and remove unused ones - rules_setts = { - 'module': module_name, - 'buildpath': build_dir, - 'dorestdoc': args.rest_doc, - 'dolatexdoc': args.latex_doc, - 'shortlatex': args.short_latex, - 'verbose': args.verbose, - 'do-lower': args.lower, - 'f2cmap_file': args.f2cmap, - 'include_paths': args.include_paths, - 'coutput': None, - 'f2py_wrapper_output': None, - 'emptygen': True, - } - crackfortran_setts = { - 'module': module_name, - 'skipfuncs': skip_funcs, - 'onlyfuncs': only_funcs, - 'verbose': args.verbose, - 'include_paths': args.include_paths, - 'do-lower': args.lower, - 'debug': args.debug_api, - 'wrapfuncs': args.wrap_functions, - } - capi_maps_setts = { - 'f2cmap': args.f2cmap, - 'headers': headers, - } - auxfuncs_setts = { - 'verbose': args.verbose, - 'debug': args.debug_api, - 'wrapfuncs': args.wrap_functions, - } - - wrapper_settings(rules_setts, crackfortran_setts, capi_maps_setts, auxfuncs_setts) - - # Disutils receives all the options and builds the extension. - if(args.c): - remove_build_dir = not bool(args.build_dir) - link_resource = args.link_resource - f2py_flags = get_f2pyflags_dist(args, skip_funcs, only_funcs, rem) - fc_flags = get_fortran_compiler_flags(args) - flib_flags = get_fortran_library_flags(args) - - ext_args = { - 'name': module_name, - 'sources': pyf_files + f77_files + f90_files, - 'include_dirs': args.include_dirs, - 'library_dirs': args.library_path, - 'libraries': args.library_name, - 'define_macros': args.define_macros, - 'undef_macros': args.undef_macros, - 'extra_objects': obj_files, - 'f2py_options': f2py_flags, + with open_build_dir(args.build_dir, args.c) as build_dir: + breakpoint() + module_name = get_module_name(args, pyf_files) + sign_file = get_signature_file(args, build_dir) + headers = get_additional_headers(rem) + # TODO: Refine rules settings. Read codebase and remove unused ones + rules_setts = { + 'module': module_name, + 'buildpath': build_dir, + 'dorestdoc': args.rest_doc, + 'dolatexdoc': args.latex_doc, + 'shortlatex': args.short_latex, + 'verbose': args.verbose, + 'do-lower': args.lower, + 'f2cmap_file': args.f2cmap, + 'include_paths': args.include_paths, + 'coutput': None, + 'f2py_wrapper_output': None, + 'emptygen': True, } - compile_dist(ext_args, link_resource, build_dir, fc_flags, flib_flags, args.quiet, remove_build_dir) - else: - generate_files(f77_files + f90_files, module_name, sign_file) + crackfortran_setts = { + 'module': module_name, + 'skipfuncs': skip_funcs, + 'onlyfuncs': only_funcs, + 'verbose': args.verbose, + 'include_paths': args.include_paths, + 'do-lower': args.lower, + 'debug': args.debug_api, + 'wrapfuncs': args.wrap_functions, + } + capi_maps_setts = { + 'f2cmap': args.f2cmap, + 'headers': headers, + } + auxfuncs_setts = { + 'verbose': args.verbose, + 'debug': args.debug_api, + 'wrapfuncs': args.wrap_functions, + } + + wrapper_settings(rules_setts, crackfortran_setts, capi_maps_setts, auxfuncs_setts) + + # Disutils receives all the options and builds the extension. + if(args.c): + link_resource = args.link_resource + f2py_flags = get_f2pyflags_dist(args, skip_funcs, only_funcs, rem) + fc_flags = get_fortran_compiler_flags(args) + flib_flags = get_fortran_library_flags(args) + + ext_args = { + 'name': module_name, + 'sources': pyf_files + f77_files + f90_files, + 'include_dirs': args.include_dirs, + 'library_dirs': args.library_path, + 'libraries': args.library_name, + 'define_macros': args.define_macros, + 'undef_macros': args.undef_macros, + 'extra_objects': obj_files, + 'f2py_options': f2py_flags, + } + compile_dist(ext_args, link_resource, build_dir, fc_flags, flib_flags, args.quiet) + else: + generate_files(f77_files + f90_files, module_name, sign_file) def main(): logger = logging.getLogger("f2py_cli") diff --git a/numpy/f2py/service.py b/numpy/f2py/service.py index 110214abcc62..7232c2a1e0b5 100644 --- a/numpy/f2py/service.py +++ b/numpy/f2py/service.py @@ -278,7 +278,7 @@ def generate_files(files: List[Path], module_name: str, sign_file: str): if(module_name): _buildmodules(postlist) -def compile_dist(ext_args, link_resources, build_dir, fc_flags, flib_flags, quiet_build, remove_build_dir=True): +def compile_dist(ext_args, link_resources, build_dir, fc_flags, flib_flags, quiet_build): _set_dependencies_dist(ext_args, link_resources) f2py_dir = get_f2py_dir() ext = Extension(**ext_args) @@ -292,10 +292,6 @@ def compile_dist(ext_args, link_resources, build_dir, fc_flags, flib_flags, quie if flib_flags: f2py_build_flags.extend(['build_ext'] + flib_flags) setup(ext_modules=[ext], script_name=f2py_dir, script_args=f2py_build_flags) - if(remove_build_dir and os.path.exists(build_dir)): - import shutil - outmess('Removing build directory %s\n' % (build_dir)) - shutil.rmtree(build_dir) def segregate_files(files: List[str]) -> Tuple[List[Path], List[Path], List[Path], List[Path]]: """ diff --git a/numpy/f2py/utils.py b/numpy/f2py/utils.py index 71298cf047d8..bb21057ac581 100644 --- a/numpy/f2py/utils.py +++ b/numpy/f2py/utils.py @@ -1,7 +1,29 @@ """Global f2py utilities.""" import os +import contextlib +import tempfile +import pathlib +import shutil def get_f2py_dir(): """Return the directory where f2py is installed.""" - return os.path.dirname(os.path.abspath(__file__)) \ No newline at end of file + return os.path.dirname(os.path.abspath(__file__)) + +@contextlib.contextmanager +def open_build_dir(build_dir: list or None, compile: bool): + remove_build_dir: bool = False + if(type(build_dir) is list and build_dir[0] is not None): + build_dir = build_dir[0] + try: + if build_dir is None: + if(compile): + remove_build_dir = True + build_dir = tempfile.mkdtemp() + else: + build_dir = pathlib.Path.cwd() + else: + build_dir = pathlib.Path(build_dir).absolute() + yield build_dir + finally: + shutil.rmtree(build_dir) if remove_build_dir else None \ No newline at end of file From d0f7d3eb5de97f2a4ed95f70ec477288d6111f8e Mon Sep 17 00:00:00 2001 From: Namami Shanker Date: Tue, 12 Jul 2022 20:05:42 +0530 Subject: [PATCH 060/132] MAINT: Remove whitespaces --- numpy/f2py/service.py | 1 - 1 file changed, 1 deletion(-) diff --git a/numpy/f2py/service.py b/numpy/f2py/service.py index 7232c2a1e0b5..bf9b7d455bc5 100644 --- a/numpy/f2py/service.py +++ b/numpy/f2py/service.py @@ -160,7 +160,6 @@ def _dict_append(d_out, d_in): else: d_out[k].append(v) - def _buildmodules(lst): cfuncs.buildcfuncs() outmess('Building modules...\n') From 68b79ebda971329d76e9491e908f36d4da78e716 Mon Sep 17 00:00:00 2001 From: Namami Shanker Date: Tue, 12 Jul 2022 20:06:15 +0530 Subject: [PATCH 061/132] FIX: Return after generating signature --- numpy/f2py/service.py | 1 + 1 file changed, 1 insertion(+) diff --git a/numpy/f2py/service.py b/numpy/f2py/service.py index bf9b7d455bc5..3ca39f047ed0 100644 --- a/numpy/f2py/service.py +++ b/numpy/f2py/service.py @@ -274,6 +274,7 @@ def generate_files(files: List[Path], module_name: str, sign_file: str): _check_postlist(postlist, sign_file) if(sign_file): _generate_signature(postlist, sign_file) + return if(module_name): _buildmodules(postlist) From 691059e0b1827a6eaccad8cd637241c685550f07 Mon Sep 17 00:00:00 2001 From: Namami Shanker Date: Tue, 12 Jul 2022 20:33:17 +0530 Subject: [PATCH 062/132] FIX: Return string path of files --- numpy/f2py/service.py | 1 - 1 file changed, 1 deletion(-) diff --git a/numpy/f2py/service.py b/numpy/f2py/service.py index 3ca39f047ed0..61371a6121bd 100644 --- a/numpy/f2py/service.py +++ b/numpy/f2py/service.py @@ -312,7 +312,6 @@ def segregate_files(files: List[str]) -> Tuple[List[Path], List[Path], List[Path other_files = [] for f in files: - f = Path(f) ext = os.path.splitext(f)[1] if ext in f77_ext: f77_files.append(f) From 21cfaf166572e6e3d7f7e8ebab9c9008648bd14a Mon Sep 17 00:00:00 2001 From: Namami Shanker Date: Tue, 12 Jul 2022 20:58:48 +0530 Subject: [PATCH 063/132] FIX: Remove sneaky breakpoint --- numpy/f2py/f2pyarg.py | 1 - 1 file changed, 1 deletion(-) diff --git a/numpy/f2py/f2pyarg.py b/numpy/f2py/f2pyarg.py index 80c59192ae96..9730c642ec9a 100644 --- a/numpy/f2py/f2pyarg.py +++ b/numpy/f2py/f2pyarg.py @@ -698,7 +698,6 @@ def process_args(args, rem): f77_files, f90_files, pyf_files, obj_files, other_files = segregate_files(files) with open_build_dir(args.build_dir, args.c) as build_dir: - breakpoint() module_name = get_module_name(args, pyf_files) sign_file = get_signature_file(args, build_dir) headers = get_additional_headers(rem) From 30748db32f188a4d0cb7ffae6629d05c3b668877 Mon Sep 17 00:00:00 2001 From: Namami Shanker Date: Tue, 12 Jul 2022 21:01:28 +0530 Subject: [PATCH 064/132] MAINT: Remove unused build_dir function --- numpy/f2py/f2pyarg.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/numpy/f2py/f2pyarg.py b/numpy/f2py/f2pyarg.py index 9730c642ec9a..dfc089beee21 100644 --- a/numpy/f2py/f2pyarg.py +++ b/numpy/f2py/f2pyarg.py @@ -647,12 +647,6 @@ def get_fortran_compiler_flags(args): if(args.debug): fc_flags.append('--debug') -def get_build_dir(args): - if(args.build_dir is not None): - return args.build_dir[0] - if(args.c): - return tempfile.mkdtemp() - return pathlib.Path.cwd() def get_module_name(args, pyf_files): if(args.module is not None): From 5539245a2d81b14bdf236f2e958fa5ff7fe41a69 Mon Sep 17 00:00:00 2001 From: Namami Shanker Date: Tue, 12 Jul 2022 22:45:10 +0530 Subject: [PATCH 065/132] FIX: Processing macros --- numpy/f2py/f2pyarg.py | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/numpy/f2py/f2pyarg.py b/numpy/f2py/f2pyarg.py index dfc089beee21..8789b4bb4ce0 100644 --- a/numpy/f2py/f2pyarg.py +++ b/numpy/f2py/f2pyarg.py @@ -134,7 +134,14 @@ def __call__(self, parser, namespace, values, option_string=None): raise RuntimeError(f"{outvar} is not in {debug_api}") class ProcessMacros(argparse.Action): - """Process macros in the form of --Dmacro=value and -Dmacro""" + """Process macros in the form of -Dmacro=value and -Dmacro""" + + def __init__(self, option_strings, dest, nargs="*", **kwargs): + """Initialization of the boolean flag + + Mimics the parent + """ + super(ProcessMacros, self).__init__(option_strings, dest, nargs="*", **kwargs) def __call__(self, parser, namespace, values, option_string=None): """The storage action @@ -143,11 +150,11 @@ def __call__(self, parser, namespace, values, option_string=None): """ items = getattr(namespace, self.dest) or [] - outvar = option_string.split("-D")[1] - if('=' in outvar): - items.append((outvar.split("=")[0], outvar.split("=")[1])) - else: - items.append((outvar, None)) + for value in values: + if('=' in value): + items.append((value.split("=")[0], value.split("=")[1])) + else: + items.append((value, None)) setattr(namespace, self.dest, items) @@ -513,9 +520,10 @@ def __call__(self, parser, namespace, values, option_string=None): build_helpers.add_argument( "-D", type=str, + metavar='MACRO[=value]', nargs="*", action=ProcessMacros, - dest='define_macros', + dest="define_macros", help="Define macros" ) From 8226050a191455d933855f44322befba4b0c8f87 Mon Sep 17 00:00:00 2001 From: Namami Shanker Date: Tue, 12 Jul 2022 22:45:51 +0530 Subject: [PATCH 066/132] FIX: f2cmap flag picking up positional args --- numpy/f2py/f2pyarg.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/numpy/f2py/f2pyarg.py b/numpy/f2py/f2pyarg.py index 8789b4bb4ce0..a926e6e6d89a 100644 --- a/numpy/f2py/f2pyarg.py +++ b/numpy/f2py/f2pyarg.py @@ -358,8 +358,6 @@ def __call__(self, parser, namespace, values, option_string=None): parser.add_argument( "--f2cmap", metavar="", - action="extend", - nargs="*", type=pathlib.Path, default=".f2py_f2cmap", help="""Load Fortran-to-Python KIND specification from the given From 4f9c84774509c220646f0b7270f1cc03f86bceef Mon Sep 17 00:00:00 2001 From: Namami Shanker Date: Tue, 12 Jul 2022 23:05:07 +0530 Subject: [PATCH 067/132] FIX: Print f2py version --- numpy/f2py/f2pyarg.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/numpy/f2py/f2pyarg.py b/numpy/f2py/f2pyarg.py index a926e6e6d89a..04764c18b67a 100644 --- a/numpy/f2py/f2pyarg.py +++ b/numpy/f2py/f2pyarg.py @@ -19,12 +19,12 @@ import argparse import logging import pathlib -import tempfile from numpy.version import version as __version__ from .service import check_dccomp, check_npfcomp, check_dir, generate_files, segregate_files, get_f2py_modulename, wrapper_settings, compile_dist from .utils import open_build_dir +from .auxfuncs import outmess ################## # Temp Variables # @@ -380,6 +380,7 @@ def __call__(self, parser, namespace, values, option_string=None): parser.add_argument( "-v", action="store_true", + dest="version", help="Print f2py version ID and exit.", ) @@ -694,6 +695,9 @@ def process_args(args, rem): if args.help: parser.print_help() parser.exit() + if(args.version): + outmess(__version__) + parser.exit() files, skip_funcs, only_funcs = segregate_posn_args(args) f77_files, f90_files, pyf_files, obj_files, other_files = segregate_files(files) From f4ab7fa6631f933338da29a518df5f5c3dab3c90 Mon Sep 17 00:00:00 2001 From: Namami Shanker Date: Wed, 13 Jul 2022 13:39:27 +0530 Subject: [PATCH 068/132] TST: Add f2pyarg modules to public api list --- numpy/tests/test_public_api.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/numpy/tests/test_public_api.py b/numpy/tests/test_public_api.py index 882a48d2ff4e..7eea1eb2fdc4 100644 --- a/numpy/tests/test_public_api.py +++ b/numpy/tests/test_public_api.py @@ -258,6 +258,9 @@ def test_NPY_NO_EXPORT(): "f2py.rules", "f2py.symbolic", "f2py.use_rules", + "f2py.f2pyarg", + "f2py.service", + "f2py.utils", "fft.helper", "lib.arraypad", "lib.arraysetops", From 7782efee9d645a7c7474fe3e941edbef8396bd81 Mon Sep 17 00:00:00 2001 From: Namami Shanker Date: Wed, 13 Jul 2022 18:20:12 +0530 Subject: [PATCH 069/132] ENH: Add f2cmap support to f2pyarg compilation --- numpy/f2py/f2pyarg.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/numpy/f2py/f2pyarg.py b/numpy/f2py/f2pyarg.py index 04764c18b67a..e2215a2263e3 100644 --- a/numpy/f2py/f2pyarg.py +++ b/numpy/f2py/f2pyarg.py @@ -621,6 +621,8 @@ def get_f2pyflags_dist(args, skip_funcs, only_funcs, rem): f2py_flags.extend(['only:']+only_funcs + [':']) if(args.include_paths): f2py_flags.extend(['--include-paths']+args.include_paths) + if(args.f2cmap): + f2py_flags.extend(['--f2cmap', str(args.f2cmap)]) return f2py_flags def get_fortran_library_flags(args): From 51290b1e89c6783d102eedba02a666bcd2906186 Mon Sep 17 00:00:00 2001 From: Namami Shanker Date: Sun, 24 Jul 2022 13:54:04 +0530 Subject: [PATCH 070/132] ENH: Add typing support --- numpy/f2py/service.py | 42 +++++++++++++++++++++--------------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/numpy/f2py/service.py b/numpy/f2py/service.py index 61371a6121bd..95526f0a9a17 100644 --- a/numpy/f2py/service.py +++ b/numpy/f2py/service.py @@ -4,7 +4,7 @@ import re from pathlib import Path -from typing import Any, Dict, List, Tuple +from typing import Any from argparse import Namespace # Distutil dependencies @@ -31,7 +31,7 @@ _f2py_user_module_name_match = re.compile(r'\s*python\s*module\s*(?P[\w_]*?' r'__user__[\w_]*)', re.I).match -def check_fortran(fname: str): +def check_fortran(fname: str) -> Path: """Function which checks This is meant as a sanity check, but will not raise an error, just a @@ -56,7 +56,7 @@ def check_fortran(fname: str): return fpname -def check_dir(dname: str): +def check_dir(dname: str) -> Path: """Function which checks the build directory This is meant to ensure no odd directories are passed, it will fail if a @@ -78,7 +78,7 @@ def check_dir(dname: str): return dpname -def check_dccomp(opt: str): +def check_dccomp(opt: str) -> str: """Function which checks for an np.distutils compliant c compiler Meant to enforce sanity checks, note that this just checks against distutils.show_compilers() @@ -100,7 +100,7 @@ def check_dccomp(opt: str): raise RuntimeError(f"{opt} is not an distutils supported C compiler, choose from {cchoices}") -def check_npfcomp(opt: str): +def check_npfcomp(opt: str) -> str: """Function which checks for an np.distutils compliant fortran compiler Meant to enforce sanity checks @@ -124,12 +124,12 @@ def check_npfcomp(opt: str): raise RuntimeError(f"{opt} is not an np.distutils supported compiler, choose from {fchoices}") -def _set_additional_headers(headers: List[str]): +def _set_additional_headers(headers: list[str]) -> None: for header in headers: cfuncs.outneeds['userincludes'].append(header[1:-1]) cfuncs.userincludes[header[1:-1]] = f"#include {header}" -def _set_crackfortran(crackfortran_setts): +def _set_crackfortran(crackfortran_setts: dict[str, str or list[str]]) -> None: crackfortran.reset_global_f2py_vars() crackfortran.f77modulename = crackfortran_setts["module"] crackfortran.include_paths[:] = crackfortran_setts["include_paths"] @@ -139,19 +139,19 @@ def _set_crackfortran(crackfortran_setts): crackfortran.onlyfuncs = crackfortran_setts["onlyfuncs"] crackfortran.dolowercase = crackfortran_setts["do-lower"] -def _set_rules(rules_setts): +def _set_rules(rules_setts: dict[str, str or list[str]]) -> None: rules.options = rules_setts -def _set_capi_maps(capi_maps_setts): +def _set_capi_maps(capi_maps_setts: dict[str, str or list[str]]) -> None: capi_maps.load_f2cmap_file(capi_maps_setts["f2cmap"]) _set_additional_headers(capi_maps_setts["headers"]) -def _set_auxfuncs(aux_funcs_setts): +def _set_auxfuncs(aux_funcs_setts: dict[str, str or list[str]]) -> None: auxfuncs.options = {'verbose': aux_funcs_setts['verbose']} auxfuncs.debugoptions = aux_funcs_setts["debug"] auxfuncs.wrapfuncs = aux_funcs_setts['wrapfuncs'] -def _dict_append(d_out, d_in): +def _dict_append(d_out: dict[str, Any], d_in: dict[str, Any]) -> None: for (k, v) in d_in.items(): if k not in d_out: d_out[k] = [] @@ -160,7 +160,7 @@ def _dict_append(d_out, d_in): else: d_out[k].append(v) -def _buildmodules(lst): +def _buildmodules(lst: list[dict[str, Any]]) -> dict[str, Any]: cfuncs.buildcfuncs() outmess('Building modules...\n') modules, mnames, isusedby = [], [], {} @@ -195,7 +195,7 @@ def _buildmodules(lst): return ret -def _generate_signature(postlist, sign_file: str): +def _generate_signature(postlist: list[dict[str, Any]], sign_file: str) -> None: outmess(f"Saving signatures to file {sign_file}" + "\n") pyf = crackfortran.crack2fortran(postlist) if sign_file in {"-", "stdout"}: @@ -204,7 +204,7 @@ def _generate_signature(postlist, sign_file: str): with open(sign_file, "w") as f: f.write(pyf) -def _check_postlist(postlist, sign_file: str): +def _check_postlist(postlist: list[dict[str, Any]], sign_file: str) -> None: isusedby = {} for plist in postlist: if 'use' in plist: @@ -233,14 +233,14 @@ def _check_postlist(postlist, sign_file: str): # raise TypeError('All blocks must be python module blocks but got %s' % ( # repr(postlist[i]['block']))) -def _callcrackfortran(files: List[Path], module_name: str): +def _callcrackfortran(files: list[str], module_name: str) -> list[dict[str, Any]]: postlist = crackfortran.crackfortran([str(file) for file in files]) for mod in postlist: mod["coutput"] = f"{module_name}module.c" mod["f2py_wrapper_output"] = f"{module_name}-f2pywrappers.f" return postlist -def _set_dependencies_dist(ext_args, link_resource): +def _set_dependencies_dist(ext_args: dict[str, Any], link_resource: list[str]) -> None: for dep in link_resource: info = get_info(dep) if not info: @@ -248,7 +248,7 @@ def _set_dependencies_dist(ext_args, link_resource): ' (try `f2py --help-link`)\n' % (repr(n))) dict_append(ext_args, **link_resource) -def get_f2py_modulename(source): +def get_f2py_modulename(source: str) -> str: name = None _f2py_module_name_match = re.compile(r'\s*python\s*module\s*(?P[\w_]+)', re.I).match @@ -263,13 +263,13 @@ def get_f2py_modulename(source): break return name -def wrapper_settings(rules_setts, crackfortran_setts, capi_maps_setts, auxfuncs_setts): +def wrapper_settings(rules_setts: dict[str, Any], crackfortran_setts: dict[str, Any], capi_maps_setts: dict[str, Any], auxfuncs_setts: dict[str, Any]) -> None: _set_rules(rules_setts) _set_crackfortran(crackfortran_setts) _set_capi_maps(capi_maps_setts) _set_auxfuncs(auxfuncs_setts) -def generate_files(files: List[Path], module_name: str, sign_file: str): +def generate_files(files: list[Path], module_name: str, sign_file: str) -> None: postlist = _callcrackfortran(files, module_name) _check_postlist(postlist, sign_file) if(sign_file): @@ -278,7 +278,7 @@ def generate_files(files: List[Path], module_name: str, sign_file: str): if(module_name): _buildmodules(postlist) -def compile_dist(ext_args, link_resources, build_dir, fc_flags, flib_flags, quiet_build): +def compile_dist(ext_args: dict[str, Any], link_resources: list[str], build_dir: Path, fc_flags: list[str], flib_flags: list[str], quiet_build: bool) -> None: _set_dependencies_dist(ext_args, link_resources) f2py_dir = get_f2py_dir() ext = Extension(**ext_args) @@ -293,7 +293,7 @@ def compile_dist(ext_args, link_resources, build_dir, fc_flags, flib_flags, quie f2py_build_flags.extend(['build_ext'] + flib_flags) setup(ext_modules=[ext], script_name=f2py_dir, script_args=f2py_build_flags) -def segregate_files(files: List[str]) -> Tuple[List[Path], List[Path], List[Path], List[Path]]: +def segregate_files(files: list[str]) -> tuple[list[str], list[str], list[str], list[str]]: """ Segregate files into three groups: * .f files From 69d3a2fe2cd7d18f1a63674ae87abc874dcfdbfb Mon Sep 17 00:00:00 2001 From: Namami Shanker Date: Sun, 24 Jul 2022 14:10:00 +0530 Subject: [PATCH 071/132] ENH: Add typing support to f2pyarg --- numpy/f2py/f2pyarg.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/numpy/f2py/f2pyarg.py b/numpy/f2py/f2pyarg.py index e2215a2263e3..5427a2a86159 100644 --- a/numpy/f2py/f2pyarg.py +++ b/numpy/f2py/f2pyarg.py @@ -594,10 +594,10 @@ def __call__(self, parser, namespace, values, option_string=None): # Main Process # ################ -def get_additional_headers(rem): +def get_additional_headers(rem: list[str]) -> list[str]: return [val[8:] for val in rem if val[:8] == '-include'] -def get_f2pyflags_dist(args, skip_funcs, only_funcs, rem): +def get_f2pyflags_dist(args: argparse.Namespace, skip_funcs: list[str], only_funcs: list[str]) -> list[str]: # Distutils requires 'f2py_options' which will be a subset of # sys.argv array received. This function reconstructs the array # from received args. @@ -625,7 +625,7 @@ def get_f2pyflags_dist(args, skip_funcs, only_funcs, rem): f2py_flags.extend(['--f2cmap', str(args.f2cmap)]) return f2py_flags -def get_fortran_library_flags(args): +def get_fortran_library_flags(args: argparse.Namespace) -> list[str]: flib_flags = [] if args.fcompiler: flib_flags.append(f'--fcompiler={args.fcompiler[0]}') @@ -633,7 +633,7 @@ def get_fortran_library_flags(args): flib_flags.append(f'--compiler={args.compiler[0]}') return flib_flags -def get_fortran_compiler_flags(args): +def get_fortran_compiler_flags(args: argparse.Namespace) -> list[str]: fc_flags = [] if(args.help_fcompiler): fc_flags.append('--help-fcompiler') @@ -657,7 +657,7 @@ def get_fortran_compiler_flags(args): fc_flags.append('--debug') -def get_module_name(args, pyf_files): +def get_module_name(args: argparse.Namespace, pyf_files: list[str]) -> str: if(args.module is not None): return args.module[0] if args.c: @@ -667,7 +667,7 @@ def get_module_name(args, pyf_files): return "unititled" return None -def get_signature_file(args, build_dir): +def get_signature_file(args: argparse.Namespace, build_dir: pathlib.Path) -> pathlib.Path: sign_file = None if(args.hint_signature): sign_file = build_dir / args.hint_signature[0] @@ -676,7 +676,7 @@ def get_signature_file(args, build_dir): parser.exit() return sign_file -def segregate_posn_args(args): +def segregate_posn_args(args: argparse.Namespace) -> tuple[list[str], list[str], list[str]]: # Currently, argparse does not recognise 'skip:' and 'only:' as optional args # and clubs them all in "Fortran Files" attr. This function segregates them. funcs = {"skip:": [], "only:": []} @@ -693,7 +693,7 @@ def segregate_posn_args(args): funcs[mode].append(arg) return files, funcs['skip:'], funcs['only:'] -def process_args(args, rem): +def process_args(args: argparse.Namespace, rem: list[str]) -> None: if args.help: parser.print_help() parser.exit() From 1c8d67eb582ebb9d559368460fa6cb746383e322 Mon Sep 17 00:00:00 2001 From: Namami Shanker Date: Sun, 24 Jul 2022 14:10:35 +0530 Subject: [PATCH 072/132] FIX: Typing in f2py service file --- numpy/f2py/service.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/numpy/f2py/service.py b/numpy/f2py/service.py index 95526f0a9a17..615662b307cb 100644 --- a/numpy/f2py/service.py +++ b/numpy/f2py/service.py @@ -195,7 +195,7 @@ def _buildmodules(lst: list[dict[str, Any]]) -> dict[str, Any]: return ret -def _generate_signature(postlist: list[dict[str, Any]], sign_file: str) -> None: +def _generate_signature(postlist: list[dict[str, Any]], sign_file: Path) -> None: outmess(f"Saving signatures to file {sign_file}" + "\n") pyf = crackfortran.crack2fortran(postlist) if sign_file in {"-", "stdout"}: @@ -204,7 +204,7 @@ def _generate_signature(postlist: list[dict[str, Any]], sign_file: str) -> None: with open(sign_file, "w") as f: f.write(pyf) -def _check_postlist(postlist: list[dict[str, Any]], sign_file: str) -> None: +def _check_postlist(postlist: list[dict[str, Any]], sign_file: Path) -> None: isusedby = {} for plist in postlist: if 'use' in plist: @@ -269,7 +269,7 @@ def wrapper_settings(rules_setts: dict[str, Any], crackfortran_setts: dict[str, _set_capi_maps(capi_maps_setts) _set_auxfuncs(auxfuncs_setts) -def generate_files(files: list[Path], module_name: str, sign_file: str) -> None: +def generate_files(files: list[Path], module_name: str, sign_file: Path) -> None: postlist = _callcrackfortran(files, module_name) _check_postlist(postlist, sign_file) if(sign_file): @@ -293,7 +293,7 @@ def compile_dist(ext_args: dict[str, Any], link_resources: list[str], build_dir: f2py_build_flags.extend(['build_ext'] + flib_flags) setup(ext_modules=[ext], script_name=f2py_dir, script_args=f2py_build_flags) -def segregate_files(files: list[str]) -> tuple[list[str], list[str], list[str], list[str]]: +def segregate_files(files: list[str]) -> tuple[list[str], list[str], list[str], list[str], list[str]]: """ Segregate files into three groups: * .f files From 8cec8623371ee91ce60f6395ee27a51fa4d37730 Mon Sep 17 00:00:00 2001 From: Namami Shanker Date: Sun, 24 Jul 2022 14:12:18 +0530 Subject: [PATCH 073/132] MAINT: Remove unnecessary imports --- numpy/f2py/service.py | 1 - 1 file changed, 1 deletion(-) diff --git a/numpy/f2py/service.py b/numpy/f2py/service.py index 615662b307cb..aa2691745a63 100644 --- a/numpy/f2py/service.py +++ b/numpy/f2py/service.py @@ -5,7 +5,6 @@ from pathlib import Path from typing import Any -from argparse import Namespace # Distutil dependencies from numpy.distutils.misc_util import dict_append From 1b2d883f98b81a347677a6372ad174e9b1db5b91 Mon Sep 17 00:00:00 2001 From: Namami Shanker Date: Sun, 24 Jul 2022 14:46:49 +0530 Subject: [PATCH 074/132] FIX: Add pyf.src file support to f2pyarg --- numpy/f2py/service.py | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/numpy/f2py/service.py b/numpy/f2py/service.py index aa2691745a63..b5af1cbcb389 100644 --- a/numpy/f2py/service.py +++ b/numpy/f2py/service.py @@ -3,7 +3,7 @@ import logging import re -from pathlib import Path +from pathlib import Path, PurePath from typing import Any # Distutil dependencies @@ -294,14 +294,16 @@ def compile_dist(ext_args: dict[str, Any], link_resources: list[str], build_dir: def segregate_files(files: list[str]) -> tuple[list[str], list[str], list[str], list[str], list[str]]: """ - Segregate files into three groups: - * .f files - * .o files + Segregate files into five groups: + * Fortran 77 files + * Fortran 90 and above files + * F2PY Signature files + * Object files * others """ f77_ext = ('.f', '.for', '.ftn', '.f77') f90_ext = ('.f90', '.f95', '.f03', '.f08') - pyf_ext = ('.pyf',) + pyf_ext = ('.pyf', '.src') out_ext = ('.o', '.out', '.so', '.a') f77_files = [] @@ -311,16 +313,18 @@ def segregate_files(files: list[str]) -> tuple[list[str], list[str], list[str], other_files = [] for f in files: - ext = os.path.splitext(f)[1] + f = PurePath(f) + ext = f.suffix if ext in f77_ext: - f77_files.append(f) + f77_files.append(f.name) elif ext in f90_ext: - f90_files.append(f) + f90_files.append(f.name) elif ext in out_ext: - out_files.append(f) + out_files.append(f.name) elif ext in pyf_ext: - pyf_files.append(f) + if ext == '.src' and f.stem.endswith('.pyf') or ext != '.src': + pyf_files.append(f.name) else: - other_files.append(f) + other_files.append(f.name) return f77_files, f90_files, pyf_files, out_files, other_files \ No newline at end of file From 8217138452b58e1994bdda6bfb5077beb7a8fac5 Mon Sep 17 00:00:00 2001 From: Namami Shanker Date: Sun, 24 Jul 2022 14:47:59 +0530 Subject: [PATCH 075/132] FIX: Remove os.path dependency from f2pyarg --- numpy/f2py/f2pyarg.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/numpy/f2py/f2pyarg.py b/numpy/f2py/f2pyarg.py index 5427a2a86159..1614a02178f8 100644 --- a/numpy/f2py/f2pyarg.py +++ b/numpy/f2py/f2pyarg.py @@ -15,7 +15,6 @@ implementation in terms of subparsers """ -import os import argparse import logging import pathlib @@ -671,7 +670,7 @@ def get_signature_file(args: argparse.Namespace, build_dir: pathlib.Path) -> pat sign_file = None if(args.hint_signature): sign_file = build_dir / args.hint_signature[0] - if sign_file and os.path.isfile(sign_file) and not args.overwrite_signature: + if sign_file and sign_file.is_file() and not args.overwrite_signature: print(f'Signature file "{sign_file}" exists!!! Use --overwrite-signature to overwrite.') parser.exit() return sign_file From b3823b15aa7712b6abddffae8020fdf3cfebbcdb Mon Sep 17 00:00:00 2001 From: Namami Shanker Date: Sun, 24 Jul 2022 14:48:25 +0530 Subject: [PATCH 076/132] FIX: Remove os.path dependency from f2py service --- numpy/f2py/service.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/numpy/f2py/service.py b/numpy/f2py/service.py index b5af1cbcb389..206f83503377 100644 --- a/numpy/f2py/service.py +++ b/numpy/f2py/service.py @@ -1,5 +1,4 @@ import sys -import os import logging import re @@ -222,7 +221,7 @@ def _check_postlist(postlist: list[dict[str, Any]], sign_file: Path) -> None: outmess( 'Stopping. Edit the signature file and then run f2py on the signature file: ') outmess('%s %s\n' % - (os.path.basename(sys.argv[0]), sign_file)) + (PurePath(sys.argv[0]).name, sign_file)) return for plist in postlist: if plist['block'] != 'python module': From 499c45cc27bab7af18a4b6d12dfe2f2ca6f88a48 Mon Sep 17 00:00:00 2001 From: Namami Shanker Date: Sun, 24 Jul 2022 15:13:33 +0530 Subject: [PATCH 077/132] FIX: Replace os.path with pathlib.Path --- numpy/f2py/capi_maps.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/numpy/f2py/capi_maps.py b/numpy/f2py/capi_maps.py index b0022b11747b..66201c3aae44 100644 --- a/numpy/f2py/capi_maps.py +++ b/numpy/f2py/capi_maps.py @@ -16,7 +16,7 @@ import copy import re -import os +from pathlib import Path from .crackfortran import markoutercomma from . import cb_rules @@ -185,8 +185,8 @@ def load_f2cmap_file(f2cmap_file): if f2cmap_file is None: # Default value - f2cmap_file = '.f2py_f2cmap' - if not os.path.isfile(f2cmap_file): + f2cmap_file = Path.cwd() / Path('.f2py_f2cmap') + if not f2cmap_file.is_file(): return # User defined additions to f2cmap_all. @@ -289,7 +289,7 @@ def getctype(var): except KeyError: errmess('getctype: "%s(kind=%s)" is mapped to C "%s" (to override define dict(%s = dict(%s="")) in %s/.f2py_f2cmap file).\n' % (typespec, var['kindselector']['kind'], ctype, - typespec, var['kindselector']['kind'], os.getcwd())) + typespec, var['kindselector']['kind'], Path.cwd())) else: if not isexternal(var): errmess('getctype: No C-type found in "%s", assuming void.\n' % var) From f20254ee40d80139260d1721aee79b6f088cf5da Mon Sep 17 00:00:00 2001 From: Namami Shanker Date: Sun, 24 Jul 2022 15:13:57 +0530 Subject: [PATCH 078/132] FIX: Typing --- numpy/f2py/service.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/numpy/f2py/service.py b/numpy/f2py/service.py index 206f83503377..20c15cf0890a 100644 --- a/numpy/f2py/service.py +++ b/numpy/f2py/service.py @@ -127,7 +127,7 @@ def _set_additional_headers(headers: list[str]) -> None: cfuncs.outneeds['userincludes'].append(header[1:-1]) cfuncs.userincludes[header[1:-1]] = f"#include {header}" -def _set_crackfortran(crackfortran_setts: dict[str, str or list[str]]) -> None: +def _set_crackfortran(crackfortran_setts: dict[str, Any]) -> None: crackfortran.reset_global_f2py_vars() crackfortran.f77modulename = crackfortran_setts["module"] crackfortran.include_paths[:] = crackfortran_setts["include_paths"] @@ -137,14 +137,14 @@ def _set_crackfortran(crackfortran_setts: dict[str, str or list[str]]) -> None: crackfortran.onlyfuncs = crackfortran_setts["onlyfuncs"] crackfortran.dolowercase = crackfortran_setts["do-lower"] -def _set_rules(rules_setts: dict[str, str or list[str]]) -> None: +def _set_rules(rules_setts: dict[str, Any]) -> None: rules.options = rules_setts -def _set_capi_maps(capi_maps_setts: dict[str, str or list[str]]) -> None: +def _set_capi_maps(capi_maps_setts: dict[str, Any]) -> None: capi_maps.load_f2cmap_file(capi_maps_setts["f2cmap"]) _set_additional_headers(capi_maps_setts["headers"]) -def _set_auxfuncs(aux_funcs_setts: dict[str, str or list[str]]) -> None: +def _set_auxfuncs(aux_funcs_setts: dict[str, Any]) -> None: auxfuncs.options = {'verbose': aux_funcs_setts['verbose']} auxfuncs.debugoptions = aux_funcs_setts["debug"] auxfuncs.wrapfuncs = aux_funcs_setts['wrapfuncs'] From 68789c5cb1f67cdda3e7f2770320f558a34845ad Mon Sep 17 00:00:00 2001 From: Namami Shanker Date: Sun, 24 Jul 2022 15:23:31 +0530 Subject: [PATCH 079/132] MAINT: Use boolean values instead of integers --- numpy/f2py/diagnose.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/numpy/f2py/diagnose.py b/numpy/f2py/diagnose.py index bbef0227e811..12acd24cbbbb 100644 --- a/numpy/f2py/diagnose.py +++ b/numpy/f2py/diagnose.py @@ -29,17 +29,17 @@ def run(): try: import numpy - has_newnumpy = 1 + has_newnumpy = True except ImportError: print('Failed to import new numpy:', sys.exc_info()[1]) - has_newnumpy = 0 + has_newnumpy = False try: from numpy.f2py import f2pyarg - has_f2pyarg = 1 + has_f2pyarg = True except ImportError: print('Failed to import f2pyarg:', sys.exc_info()[1]) - has_f2pyarg = 0 + has_f2pyarg = False try: import numpy.distutils From 91b77bf37727be203505f50048810e6c8d987ded Mon Sep 17 00:00:00 2001 From: Namami Shanker Date: Sun, 24 Jul 2022 15:35:47 +0530 Subject: [PATCH 080/132] MAINT: Refactorings --- numpy/f2py/__init__.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/numpy/f2py/__init__.py b/numpy/f2py/__init__.py index 2eefe2cfb8c3..9ce3e341b6b1 100644 --- a/numpy/f2py/__init__.py +++ b/numpy/f2py/__init__.py @@ -8,10 +8,8 @@ import subprocess import os -from . import f2pyarg from . import diagnose - -main = f2pyarg.main +from numpy.f2py.f2pyarg import main def compile(source, @@ -101,7 +99,7 @@ def compile(source, c = [sys.executable, '-c', - 'import numpy.f2py as f2pyarg;f2pyarg.main()'] + args + 'from numpy.f2py import main;main()'] + args try: cp = subprocess.run(c, stdout=subprocess.PIPE, stderr=subprocess.PIPE) From 776234156cff0ba1cff8db12b603d59ce14b5c43 Mon Sep 17 00:00:00 2001 From: Namami Shanker Date: Sun, 24 Jul 2022 15:38:55 +0530 Subject: [PATCH 081/132] MAINT: Remove unnecessary comments --- numpy/f2py/service.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/numpy/f2py/service.py b/numpy/f2py/service.py index 20c15cf0890a..1522380afe8d 100644 --- a/numpy/f2py/service.py +++ b/numpy/f2py/service.py @@ -212,7 +212,6 @@ def _check_postlist(postlist: list[dict[str, Any]], sign_file: Path) -> None: isusedby[u].append(plist['name']) for plist in postlist: if plist['block'] == 'python module' and '__user__' in plist['name'] and plist['name'] in isusedby: - # if not quiet: outmess( f'Skipping Makefile build for module "{plist["name"]}" ' 'which is used by {}\n'.format( @@ -225,11 +224,8 @@ def _check_postlist(postlist: list[dict[str, Any]], sign_file: Path) -> None: return for plist in postlist: if plist['block'] != 'python module': - # if 'python module' not in options: - outmess( - 'Tip: If your original code is Fortran source then you must use -m option.\n') - # raise TypeError('All blocks must be python module blocks but got %s' % ( - # repr(postlist[i]['block']))) + outmess( + 'Tip: If your original code is Fortran source then you must use -m option.\n') def _callcrackfortran(files: list[str], module_name: str) -> list[dict[str, Any]]: postlist = crackfortran.crackfortran([str(file) for file in files]) From 6bd3e6ca559abe3475fd80563801b5c88e7aa506 Mon Sep 17 00:00:00 2001 From: Namami Shanker Date: Sun, 24 Jul 2022 15:39:43 +0530 Subject: [PATCH 082/132] FIX: set_dependecies_dist potential bug --- numpy/f2py/service.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/numpy/f2py/service.py b/numpy/f2py/service.py index 1522380afe8d..e9195fbc6c76 100644 --- a/numpy/f2py/service.py +++ b/numpy/f2py/service.py @@ -239,7 +239,7 @@ def _set_dependencies_dist(ext_args: dict[str, Any], link_resource: list[str]) - info = get_info(dep) if not info: outmess('No %s resources found in system' - ' (try `f2py --help-link`)\n' % (repr(n))) + ' (try `f2py --help-link`)\n' % (repr(dep))) dict_append(ext_args, **link_resource) def get_f2py_modulename(source: str) -> str: From 5fd51235fb7c0a4aa8908ff988184597d44044b1 Mon Sep 17 00:00:00 2001 From: Namami Shanker Date: Sun, 24 Jul 2022 15:42:21 +0530 Subject: [PATCH 083/132] MAINT: Rename regex variables as module constants --- numpy/f2py/service.py | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/numpy/f2py/service.py b/numpy/f2py/service.py index e9195fbc6c76..c4d87d09bcb5 100644 --- a/numpy/f2py/service.py +++ b/numpy/f2py/service.py @@ -24,9 +24,9 @@ logger = logging.getLogger("f2py_cli") logger.setLevel(logging.WARNING) -_f2py_module_name_match = re.compile(r'\s*python\s*module\s*(?P[\w_]+)', +F2PY_MODULE_NAME_MATCH = re.compile(r'\s*python\s*module\s*(?P[\w_]+)', re.I).match -_f2py_user_module_name_match = re.compile(r'\s*python\s*module\s*(?P[\w_]*?' +F2PY_USER_MODULE_NAME_MATCH = re.compile(r'\s*python\s*module\s*(?P[\w_]*?' r'__user__[\w_]*)', re.I).match def check_fortran(fname: str) -> Path: @@ -244,14 +244,10 @@ def _set_dependencies_dist(ext_args: dict[str, Any], link_resource: list[str]) - def get_f2py_modulename(source: str) -> str: name = None - _f2py_module_name_match = re.compile(r'\s*python\s*module\s*(?P[\w_]+)', - re.I).match - _f2py_user_module_name_match = re.compile(r'\s*python\s*module\s*(?P[\w_]*?' - r'__user__[\w_]*)', re.I).match with open(source) as f: for line in f: - if m := _f2py_module_name_match(line): - if _f2py_user_module_name_match(line): # skip *__user__* names + if m := F2PY_MODULE_NAME_MATCH(line): + if F2PY_USER_MODULE_NAME_MATCH(line): # skip *__user__* names continue name = m.group('name') break From 911fd6a49613faf56fe5743aa625a8ae95a006ce Mon Sep 17 00:00:00 2001 From: Namami Shanker Date: Sun, 24 Jul 2022 15:47:56 +0530 Subject: [PATCH 084/132] MAINT: Replace os.path with pathlib.Path --- numpy/f2py/utils.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/numpy/f2py/utils.py b/numpy/f2py/utils.py index bb21057ac581..dc65ba9c1870 100644 --- a/numpy/f2py/utils.py +++ b/numpy/f2py/utils.py @@ -1,6 +1,5 @@ """Global f2py utilities.""" -import os import contextlib import tempfile import pathlib @@ -8,7 +7,7 @@ def get_f2py_dir(): """Return the directory where f2py is installed.""" - return os.path.dirname(os.path.abspath(__file__)) + return pathlib.Path(__file__).resolve().parent @contextlib.contextmanager def open_build_dir(build_dir: list or None, compile: bool): From dac9dd42fd4700a81b56623fad80b4cb16571ff0 Mon Sep 17 00:00:00 2001 From: Namami Shanker Date: Sun, 24 Jul 2022 15:56:22 +0530 Subject: [PATCH 085/132] MAINT: Typing and pathlib related refactoring --- numpy/f2py/utils.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/numpy/f2py/utils.py b/numpy/f2py/utils.py index dc65ba9c1870..8599465a18a1 100644 --- a/numpy/f2py/utils.py +++ b/numpy/f2py/utils.py @@ -2,15 +2,17 @@ import contextlib import tempfile -import pathlib import shutil +from typing import Optional +from pathlib import Path + def get_f2py_dir(): """Return the directory where f2py is installed.""" - return pathlib.Path(__file__).resolve().parent + return Path(__file__).resolve().parent @contextlib.contextmanager -def open_build_dir(build_dir: list or None, compile: bool): +def open_build_dir(build_dir: Optional[list[Path]], compile: bool): remove_build_dir: bool = False if(type(build_dir) is list and build_dir[0] is not None): build_dir = build_dir[0] @@ -20,9 +22,9 @@ def open_build_dir(build_dir: list or None, compile: bool): remove_build_dir = True build_dir = tempfile.mkdtemp() else: - build_dir = pathlib.Path.cwd() + build_dir = Path.cwd() else: - build_dir = pathlib.Path(build_dir).absolute() + build_dir = Path(build_dir).absolute() yield build_dir finally: shutil.rmtree(build_dir) if remove_build_dir else None \ No newline at end of file From b8911884166c5e8a06685ea6ab8a7e6cceffb714 Mon Sep 17 00:00:00 2001 From: Namami Shanker Date: Sun, 24 Jul 2022 16:17:08 +0530 Subject: [PATCH 086/132] FIX: Passing rem to f2pyflags_dist --- numpy/f2py/f2pyarg.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/numpy/f2py/f2pyarg.py b/numpy/f2py/f2pyarg.py index 1614a02178f8..907dbe84012b 100644 --- a/numpy/f2py/f2pyarg.py +++ b/numpy/f2py/f2pyarg.py @@ -746,7 +746,7 @@ def process_args(args: argparse.Namespace, rem: list[str]) -> None: # Disutils receives all the options and builds the extension. if(args.c): link_resource = args.link_resource - f2py_flags = get_f2pyflags_dist(args, skip_funcs, only_funcs, rem) + f2py_flags = get_f2pyflags_dist(args, skip_funcs, only_funcs) fc_flags = get_fortran_compiler_flags(args) flib_flags = get_fortran_library_flags(args) From 47a94268e6719dd3e1fcff034370f07194adc6e4 Mon Sep 17 00:00:00 2001 From: Namami Shanker Date: Sun, 24 Jul 2022 16:19:57 +0530 Subject: [PATCH 087/132] FIX: Always convert f2cmap_file to Path type --- numpy/f2py/capi_maps.py | 1 + 1 file changed, 1 insertion(+) diff --git a/numpy/f2py/capi_maps.py b/numpy/f2py/capi_maps.py index 66201c3aae44..2543200f6429 100644 --- a/numpy/f2py/capi_maps.py +++ b/numpy/f2py/capi_maps.py @@ -186,6 +186,7 @@ def load_f2cmap_file(f2cmap_file): if f2cmap_file is None: # Default value f2cmap_file = Path.cwd() / Path('.f2py_f2cmap') + f2cmap_file = Path(f2cmap_file) if not f2cmap_file.is_file(): return From 5772ec62814ad7d66abf2e4e8d6b6fb1109ef54d Mon Sep 17 00:00:00 2001 From: Namami Shanker Date: Sun, 24 Jul 2022 16:22:01 +0530 Subject: [PATCH 088/132] MAINT: Apply changes from code review --- numpy/f2py/utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/numpy/f2py/utils.py b/numpy/f2py/utils.py index 8599465a18a1..da81ee7ceb68 100644 --- a/numpy/f2py/utils.py +++ b/numpy/f2py/utils.py @@ -14,8 +14,8 @@ def get_f2py_dir(): @contextlib.contextmanager def open_build_dir(build_dir: Optional[list[Path]], compile: bool): remove_build_dir: bool = False - if(type(build_dir) is list and build_dir[0] is not None): - build_dir = build_dir[0] + if(isinstance(build_dir, list)): + build_dir = build_dir[0] if build_dir else None try: if build_dir is None: if(compile): From 04558471297cea6972e74ca9e4401c57fab85b04 Mon Sep 17 00:00:00 2001 From: Namami Shanker Date: Sun, 24 Jul 2022 16:23:19 +0530 Subject: [PATCH 089/132] MAINT: No need to use try..finally --- numpy/f2py/utils.py | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/numpy/f2py/utils.py b/numpy/f2py/utils.py index da81ee7ceb68..4a72926a0183 100644 --- a/numpy/f2py/utils.py +++ b/numpy/f2py/utils.py @@ -16,15 +16,13 @@ def open_build_dir(build_dir: Optional[list[Path]], compile: bool): remove_build_dir: bool = False if(isinstance(build_dir, list)): build_dir = build_dir[0] if build_dir else None - try: - if build_dir is None: - if(compile): - remove_build_dir = True - build_dir = tempfile.mkdtemp() - else: - build_dir = Path.cwd() + if build_dir is None: + if(compile): + remove_build_dir = True + build_dir = tempfile.mkdtemp() else: - build_dir = Path(build_dir).absolute() - yield build_dir - finally: - shutil.rmtree(build_dir) if remove_build_dir else None \ No newline at end of file + build_dir = Path.cwd() + else: + build_dir = Path(build_dir).absolute() + yield build_dir + shutil.rmtree(build_dir) if remove_build_dir else None \ No newline at end of file From e1f722fd27db034da913262b72a50f0d589b0b52 Mon Sep 17 00:00:00 2001 From: Namami Shanker Date: Sun, 24 Jul 2022 17:16:33 +0530 Subject: [PATCH 090/132] FIX: Return full path string --- numpy/f2py/service.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/numpy/f2py/service.py b/numpy/f2py/service.py index c4d87d09bcb5..d31f56e49906 100644 --- a/numpy/f2py/service.py +++ b/numpy/f2py/service.py @@ -304,18 +304,18 @@ def segregate_files(files: list[str]) -> tuple[list[str], list[str], list[str], other_files = [] for f in files: - f = PurePath(f) - ext = f.suffix + f_path = PurePath(f) + ext = f_path.suffix if ext in f77_ext: - f77_files.append(f.name) + f77_files.append(f) elif ext in f90_ext: - f90_files.append(f.name) + f90_files.append(f) elif ext in out_ext: - out_files.append(f.name) + out_files.append(f) elif ext in pyf_ext: - if ext == '.src' and f.stem.endswith('.pyf') or ext != '.src': - pyf_files.append(f.name) + if ext == '.src' and f_path.stem.endswith('.pyf') or ext != '.src': + pyf_files.append(f) else: - other_files.append(f.name) + other_files.append(f) return f77_files, f90_files, pyf_files, out_files, other_files \ No newline at end of file From 302c77c0616c917b5e3a83fa60c3ad13f3525ae1 Mon Sep 17 00:00:00 2001 From: Namami Shanker Date: Sun, 24 Jul 2022 17:17:03 +0530 Subject: [PATCH 091/132] FIX: Print name of PosiPath type variable --- numpy/f2py/capi_maps.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/numpy/f2py/capi_maps.py b/numpy/f2py/capi_maps.py index 2543200f6429..f8c0f4ed975b 100644 --- a/numpy/f2py/capi_maps.py +++ b/numpy/f2py/capi_maps.py @@ -196,7 +196,7 @@ def load_f2cmap_file(f2cmap_file): # interpreted as C 'float'. This feature is useful for F90/95 users if # they use PARAMETERS in type specifications. try: - outmess('Reading f2cmap from {!r} ...\n'.format(f2cmap_file)) + outmess('Reading f2cmap from {!r} ...\n'.format(f2cmap_file.name)) with open(f2cmap_file, 'r') as f: d = eval(f.read().lower(), {}, {}) for k, d1 in d.items(): From f105eecd324ab4909945a7eece8a5bed673cc808 Mon Sep 17 00:00:00 2001 From: Namami Shanker Date: Sun, 24 Jul 2022 17:40:37 +0530 Subject: [PATCH 092/132] MAINT: Refactorings from code review --- numpy/f2py/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/numpy/f2py/utils.py b/numpy/f2py/utils.py index 4a72926a0183..275d22562933 100644 --- a/numpy/f2py/utils.py +++ b/numpy/f2py/utils.py @@ -17,7 +17,7 @@ def open_build_dir(build_dir: Optional[list[Path]], compile: bool): if(isinstance(build_dir, list)): build_dir = build_dir[0] if build_dir else None if build_dir is None: - if(compile): + if compile: remove_build_dir = True build_dir = tempfile.mkdtemp() else: From 1cc1ab908501e3a53807fe57fb888bc513724572 Mon Sep 17 00:00:00 2001 From: Namami Shanker Date: Sun, 24 Jul 2022 17:46:30 +0530 Subject: [PATCH 093/132] MAINT: Refactorings from code review --- numpy/f2py/f2pyarg.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/numpy/f2py/f2pyarg.py b/numpy/f2py/f2pyarg.py index 907dbe84012b..d6243ea90e13 100644 --- a/numpy/f2py/f2pyarg.py +++ b/numpy/f2py/f2pyarg.py @@ -6,8 +6,8 @@ The concept is based around the idea that F2PY is overloaded in terms of functionality: -1. Creating the wrapper `.c` files -2. Generating `.pyf` signature files +1. Generating `.pyf` signature files +2. Creating the wrapper `.c` files 3. Compilation helpers a. This essentially means `numpy.distutils` for now From 116315e0e8efb4a286370590b79817a9bb3a6319 Mon Sep 17 00:00:00 2001 From: Namami Shanker Date: Sun, 24 Jul 2022 18:01:26 +0530 Subject: [PATCH 094/132] FIX: Typing errors --- numpy/f2py/utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/numpy/f2py/utils.py b/numpy/f2py/utils.py index 275d22562933..f0494865635c 100644 --- a/numpy/f2py/utils.py +++ b/numpy/f2py/utils.py @@ -7,12 +7,12 @@ from typing import Optional from pathlib import Path -def get_f2py_dir(): +def get_f2py_dir() -> Path: """Return the directory where f2py is installed.""" return Path(__file__).resolve().parent @contextlib.contextmanager -def open_build_dir(build_dir: Optional[list[Path]], compile: bool): +def open_build_dir(build_dir: Optional[list], compile: bool) -> Path: remove_build_dir: bool = False if(isinstance(build_dir, list)): build_dir = build_dir[0] if build_dir else None From 4b55a17d687212b3f391b0b80c4e0695fd699f37 Mon Sep 17 00:00:00 2001 From: Namami Shanker Date: Sun, 24 Jul 2022 19:12:31 +0530 Subject: [PATCH 095/132] FIX: Replace builtin types with typing python 3.8 does not support substriptable builtin types. --- numpy/f2py/f2pyarg.py | 15 +++++++------- numpy/f2py/service.py | 46 ++++++++++++++++++++++--------------------- numpy/f2py/utils.py | 4 ++-- 3 files changed, 34 insertions(+), 31 deletions(-) diff --git a/numpy/f2py/f2pyarg.py b/numpy/f2py/f2pyarg.py index d6243ea90e13..78a10ba27ca1 100644 --- a/numpy/f2py/f2pyarg.py +++ b/numpy/f2py/f2pyarg.py @@ -20,6 +20,7 @@ import pathlib from numpy.version import version as __version__ +from typing import List, Dict, Tuple, Any from .service import check_dccomp, check_npfcomp, check_dir, generate_files, segregate_files, get_f2py_modulename, wrapper_settings, compile_dist from .utils import open_build_dir @@ -593,10 +594,10 @@ def __call__(self, parser, namespace, values, option_string=None): # Main Process # ################ -def get_additional_headers(rem: list[str]) -> list[str]: +def get_additional_headers(rem: List[str]) -> List[str]: return [val[8:] for val in rem if val[:8] == '-include'] -def get_f2pyflags_dist(args: argparse.Namespace, skip_funcs: list[str], only_funcs: list[str]) -> list[str]: +def get_f2pyflags_dist(args: argparse.Namespace, skip_funcs: List[str], only_funcs: List[str]) -> List[str]: # Distutils requires 'f2py_options' which will be a subset of # sys.argv array received. This function reconstructs the array # from received args. @@ -624,7 +625,7 @@ def get_f2pyflags_dist(args: argparse.Namespace, skip_funcs: list[str], only_fun f2py_flags.extend(['--f2cmap', str(args.f2cmap)]) return f2py_flags -def get_fortran_library_flags(args: argparse.Namespace) -> list[str]: +def get_fortran_library_flags(args: argparse.Namespace) -> List[str]: flib_flags = [] if args.fcompiler: flib_flags.append(f'--fcompiler={args.fcompiler[0]}') @@ -632,7 +633,7 @@ def get_fortran_library_flags(args: argparse.Namespace) -> list[str]: flib_flags.append(f'--compiler={args.compiler[0]}') return flib_flags -def get_fortran_compiler_flags(args: argparse.Namespace) -> list[str]: +def get_fortran_compiler_flags(args: argparse.Namespace) -> List[str]: fc_flags = [] if(args.help_fcompiler): fc_flags.append('--help-fcompiler') @@ -656,7 +657,7 @@ def get_fortran_compiler_flags(args: argparse.Namespace) -> list[str]: fc_flags.append('--debug') -def get_module_name(args: argparse.Namespace, pyf_files: list[str]) -> str: +def get_module_name(args: argparse.Namespace, pyf_files: List[str]) -> str: if(args.module is not None): return args.module[0] if args.c: @@ -675,7 +676,7 @@ def get_signature_file(args: argparse.Namespace, build_dir: pathlib.Path) -> pat parser.exit() return sign_file -def segregate_posn_args(args: argparse.Namespace) -> tuple[list[str], list[str], list[str]]: +def segregate_posn_args(args: argparse.Namespace) -> Tuple[List[str], List[str], List[str]]: # Currently, argparse does not recognise 'skip:' and 'only:' as optional args # and clubs them all in "Fortran Files" attr. This function segregates them. funcs = {"skip:": [], "only:": []} @@ -692,7 +693,7 @@ def segregate_posn_args(args: argparse.Namespace) -> tuple[list[str], list[str], funcs[mode].append(arg) return files, funcs['skip:'], funcs['only:'] -def process_args(args: argparse.Namespace, rem: list[str]) -> None: +def process_args(args: argparse.Namespace, rem: List[str]) -> None: if args.help: parser.print_help() parser.exit() diff --git a/numpy/f2py/service.py b/numpy/f2py/service.py index d31f56e49906..60ca31e46dc2 100644 --- a/numpy/f2py/service.py +++ b/numpy/f2py/service.py @@ -3,7 +3,7 @@ import re from pathlib import Path, PurePath -from typing import Any +from typing import List, Dict, Tuple, Any, Optional # Distutil dependencies from numpy.distutils.misc_util import dict_append @@ -54,7 +54,7 @@ def check_fortran(fname: str) -> Path: return fpname -def check_dir(dname: str) -> Path: +def check_dir(dname: str) -> Optional[Path]: """Function which checks the build directory This is meant to ensure no odd directories are passed, it will fail if a @@ -74,6 +74,7 @@ def check_dir(dname: str) -> Path: dpname = Path(dname) dpname.mkdir(parents=True, exist_ok=True) return dpname + return None def check_dccomp(opt: str) -> str: @@ -122,12 +123,12 @@ def check_npfcomp(opt: str) -> str: raise RuntimeError(f"{opt} is not an np.distutils supported compiler, choose from {fchoices}") -def _set_additional_headers(headers: list[str]) -> None: +def _set_additional_headers(headers: List[str]) -> None: for header in headers: cfuncs.outneeds['userincludes'].append(header[1:-1]) cfuncs.userincludes[header[1:-1]] = f"#include {header}" -def _set_crackfortran(crackfortran_setts: dict[str, Any]) -> None: +def _set_crackfortran(crackfortran_setts: Dict[str, Any]) -> None: crackfortran.reset_global_f2py_vars() crackfortran.f77modulename = crackfortran_setts["module"] crackfortran.include_paths[:] = crackfortran_setts["include_paths"] @@ -137,19 +138,19 @@ def _set_crackfortran(crackfortran_setts: dict[str, Any]) -> None: crackfortran.onlyfuncs = crackfortran_setts["onlyfuncs"] crackfortran.dolowercase = crackfortran_setts["do-lower"] -def _set_rules(rules_setts: dict[str, Any]) -> None: +def _set_rules(rules_setts: Dict[str, Any]) -> None: rules.options = rules_setts -def _set_capi_maps(capi_maps_setts: dict[str, Any]) -> None: +def _set_capi_maps(capi_maps_setts: Dict[str, Any]) -> None: capi_maps.load_f2cmap_file(capi_maps_setts["f2cmap"]) _set_additional_headers(capi_maps_setts["headers"]) -def _set_auxfuncs(aux_funcs_setts: dict[str, Any]) -> None: +def _set_auxfuncs(aux_funcs_setts: Dict[str, Any]) -> None: auxfuncs.options = {'verbose': aux_funcs_setts['verbose']} auxfuncs.debugoptions = aux_funcs_setts["debug"] auxfuncs.wrapfuncs = aux_funcs_setts['wrapfuncs'] -def _dict_append(d_out: dict[str, Any], d_in: dict[str, Any]) -> None: +def _dict_append(d_out: Dict[str, Any], d_in: Dict[str, Any]) -> None: for (k, v) in d_in.items(): if k not in d_out: d_out[k] = [] @@ -158,10 +159,11 @@ def _dict_append(d_out: dict[str, Any], d_in: dict[str, Any]) -> None: else: d_out[k].append(v) -def _buildmodules(lst: list[dict[str, Any]]) -> dict[str, Any]: +def _buildmodules(lst: List[Dict[str, Any]]) -> Dict[str, Any]: cfuncs.buildcfuncs() outmess('Building modules...\n') - modules, mnames, isusedby = [], [], {} + modules, mnames = [], [] + isusedby: Dict[str, List[Any]] = {} for item in lst: if '__user__' in item['name']: cb_rules.buildcallbacks(item) @@ -173,7 +175,7 @@ def _buildmodules(lst: list[dict[str, Any]]) -> dict[str, Any]: isusedby[u].append(item['name']) modules.append(item) mnames.append(item['name']) - ret = {} + ret: Dict[str, Any] = {} for module, name in zip(modules, mnames): if name in isusedby: outmess('\tSkipping module "%s" which is used by %s.\n' % ( @@ -193,7 +195,7 @@ def _buildmodules(lst: list[dict[str, Any]]) -> dict[str, Any]: return ret -def _generate_signature(postlist: list[dict[str, Any]], sign_file: Path) -> None: +def _generate_signature(postlist: List[Dict[str, Any]], sign_file: Path) -> None: outmess(f"Saving signatures to file {sign_file}" + "\n") pyf = crackfortran.crack2fortran(postlist) if sign_file in {"-", "stdout"}: @@ -202,8 +204,8 @@ def _generate_signature(postlist: list[dict[str, Any]], sign_file: Path) -> None with open(sign_file, "w") as f: f.write(pyf) -def _check_postlist(postlist: list[dict[str, Any]], sign_file: Path) -> None: - isusedby = {} +def _check_postlist(postlist: List[Dict[str, Any]], sign_file: Path) -> None: + isusedby: Dict[str, List[Any]] = {} for plist in postlist: if 'use' in plist: for u in plist['use'].keys(): @@ -227,22 +229,22 @@ def _check_postlist(postlist: list[dict[str, Any]], sign_file: Path) -> None: outmess( 'Tip: If your original code is Fortran source then you must use -m option.\n') -def _callcrackfortran(files: list[str], module_name: str) -> list[dict[str, Any]]: +def _callcrackfortran(files: List[str], module_name: str) -> List[Dict[str, Any]]: postlist = crackfortran.crackfortran([str(file) for file in files]) for mod in postlist: mod["coutput"] = f"{module_name}module.c" mod["f2py_wrapper_output"] = f"{module_name}-f2pywrappers.f" return postlist -def _set_dependencies_dist(ext_args: dict[str, Any], link_resource: list[str]) -> None: +def _set_dependencies_dist(ext_args: Dict[str, Any], link_resource: List[str]) -> None: for dep in link_resource: info = get_info(dep) if not info: outmess('No %s resources found in system' ' (try `f2py --help-link`)\n' % (repr(dep))) - dict_append(ext_args, **link_resource) + dict_append(ext_args, **info) -def get_f2py_modulename(source: str) -> str: +def get_f2py_modulename(source: str) -> Optional[str]: name = None with open(source) as f: for line in f: @@ -253,13 +255,13 @@ def get_f2py_modulename(source: str) -> str: break return name -def wrapper_settings(rules_setts: dict[str, Any], crackfortran_setts: dict[str, Any], capi_maps_setts: dict[str, Any], auxfuncs_setts: dict[str, Any]) -> None: +def wrapper_settings(rules_setts: Dict[str, Any], crackfortran_setts: Dict[str, Any], capi_maps_setts: Dict[str, Any], auxfuncs_setts: Dict[str, Any]) -> None: _set_rules(rules_setts) _set_crackfortran(crackfortran_setts) _set_capi_maps(capi_maps_setts) _set_auxfuncs(auxfuncs_setts) -def generate_files(files: list[Path], module_name: str, sign_file: Path) -> None: +def generate_files(files: List[str], module_name: str, sign_file: Path) -> None: postlist = _callcrackfortran(files, module_name) _check_postlist(postlist, sign_file) if(sign_file): @@ -268,7 +270,7 @@ def generate_files(files: list[Path], module_name: str, sign_file: Path) -> None if(module_name): _buildmodules(postlist) -def compile_dist(ext_args: dict[str, Any], link_resources: list[str], build_dir: Path, fc_flags: list[str], flib_flags: list[str], quiet_build: bool) -> None: +def compile_dist(ext_args: Dict[str, Any], link_resources: List[str], build_dir: Path, fc_flags: List[str], flib_flags: List[str], quiet_build: bool) -> None: _set_dependencies_dist(ext_args, link_resources) f2py_dir = get_f2py_dir() ext = Extension(**ext_args) @@ -283,7 +285,7 @@ def compile_dist(ext_args: dict[str, Any], link_resources: list[str], build_dir: f2py_build_flags.extend(['build_ext'] + flib_flags) setup(ext_modules=[ext], script_name=f2py_dir, script_args=f2py_build_flags) -def segregate_files(files: list[str]) -> tuple[list[str], list[str], list[str], list[str], list[str]]: +def segregate_files(files: List[str]) -> Tuple[List[str], List[str], List[str], List[str], List[str]]: """ Segregate files into five groups: * Fortran 77 files diff --git a/numpy/f2py/utils.py b/numpy/f2py/utils.py index f0494865635c..501f4208e168 100644 --- a/numpy/f2py/utils.py +++ b/numpy/f2py/utils.py @@ -4,7 +4,7 @@ import tempfile import shutil -from typing import Optional +from typing import Optional, List from pathlib import Path def get_f2py_dir() -> Path: @@ -12,7 +12,7 @@ def get_f2py_dir() -> Path: return Path(__file__).resolve().parent @contextlib.contextmanager -def open_build_dir(build_dir: Optional[list], compile: bool) -> Path: +def open_build_dir(build_dir: Optional[List[str]], compile: bool) -> Path: remove_build_dir: bool = False if(isinstance(build_dir, list)): build_dir = build_dir[0] if build_dir else None From 232decfefe7dc78289ea8fd585c7c58486089dac Mon Sep 17 00:00:00 2001 From: Namami Shanker Date: Sun, 24 Jul 2022 19:28:51 +0530 Subject: [PATCH 096/132] MAINT: Add docstring to explain open_build_dir --- numpy/f2py/utils.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/numpy/f2py/utils.py b/numpy/f2py/utils.py index 501f4208e168..148f75467ea0 100644 --- a/numpy/f2py/utils.py +++ b/numpy/f2py/utils.py @@ -13,6 +13,13 @@ def get_f2py_dir() -> Path: @contextlib.contextmanager def open_build_dir(build_dir: Optional[List[str]], compile: bool) -> Path: + """Create build directory if the user specifies it, + Otherwise, create a temporary directory and remove it. + + Default build directory for only wrapper generation is + the current directory. Therefore, if `compile` is False, + the wrappers are generated in the current directory""" + remove_build_dir: bool = False if(isinstance(build_dir, list)): build_dir = build_dir[0] if build_dir else None From 257b44394df5f4cf6d5d2a1375b9b06fa86267cd Mon Sep 17 00:00:00 2001 From: Namami Shanker Date: Sun, 24 Jul 2022 23:41:10 +0530 Subject: [PATCH 097/132] ENH: Add temporary comments --- numpy/f2py/f2pyarg.py | 26 ++++++++++++++++++++++++++ numpy/f2py/service.py | 20 ++++++++++++++++++++ 2 files changed, 46 insertions(+) diff --git a/numpy/f2py/f2pyarg.py b/numpy/f2py/f2pyarg.py index 78a10ba27ca1..e199805c2cc4 100644 --- a/numpy/f2py/f2pyarg.py +++ b/numpy/f2py/f2pyarg.py @@ -700,14 +700,31 @@ def process_args(args: argparse.Namespace, rem: List[str]) -> None: if(args.version): outmess(__version__) parser.exit() + + # Step 1: Segregate input files from 'skip:' and 'only:' args + # Read comments in the 'segregate_posn_args' function for more detail files, skip_funcs, only_funcs = segregate_posn_args(args) + + # Step 2: Segregate source source files based on their extensions f77_files, f90_files, pyf_files, obj_files, other_files = segregate_files(files) + # Step 3: Open the correct build directory. Read 'open_build_dir' docstring for more detail with open_build_dir(args.build_dir, args.c) as build_dir: + # Step 4: Get module name and signature file path module_name = get_module_name(args, pyf_files) sign_file = get_signature_file(args, build_dir) + + # Step 5: Parse '-include
' flags and store
s in a list + # since argparse can't handle '-include
' + # we filter it out into rem and parse it manually. headers = get_additional_headers(rem) # TODO: Refine rules settings. Read codebase and remove unused ones + + # Step 6: Generate settings dictionary for f2py internal files + # The variables in `rules.py`, `crackfortran.py`, + # `capy_maps.py` and `auxfuncs.py` are set using + # information in these dictionaries. + # These are the same which 'f2py2e' passes to internal files rules_setts = { 'module': module_name, 'buildpath': build_dir, @@ -742,15 +759,23 @@ def process_args(args: argparse.Namespace, rem: List[str]) -> None: 'wrapfuncs': args.wrap_functions, } + # The function below sets the global and module variables in internal files + # Read the comments inside this function for explanation wrapper_settings(rules_setts, crackfortran_setts, capi_maps_setts, auxfuncs_setts) + # Step 7: If user has asked for compilation. Mimic 'run_compile' from f2py2e # Disutils receives all the options and builds the extension. if(args.c): link_resource = args.link_resource + + # The 3 functions below generate arrays of flag similar to how + # 'run_compile()' segregates flags into different arrays f2py_flags = get_f2pyflags_dist(args, skip_funcs, only_funcs) fc_flags = get_fortran_compiler_flags(args) flib_flags = get_fortran_library_flags(args) + # The array of flags from above is passed to distutils where + # it is handled internally ext_args = { 'name': module_name, 'sources': pyf_files + f77_files + f90_files, @@ -764,6 +789,7 @@ def process_args(args: argparse.Namespace, rem: List[str]) -> None: } compile_dist(ext_args, link_resource, build_dir, fc_flags, flib_flags, args.quiet) else: + # Step 8: Generate wrapper or signature file if compile flag is not given generate_files(f77_files + f90_files, module_name, sign_file) def main(): diff --git a/numpy/f2py/service.py b/numpy/f2py/service.py index 60ca31e46dc2..6885b58f379d 100644 --- a/numpy/f2py/service.py +++ b/numpy/f2py/service.py @@ -256,21 +256,37 @@ def get_f2py_modulename(source: str) -> Optional[str]: return name def wrapper_settings(rules_setts: Dict[str, Any], crackfortran_setts: Dict[str, Any], capi_maps_setts: Dict[str, Any], auxfuncs_setts: Dict[str, Any]) -> None: + # This function also mimics f2py2e. I have added the link to specific code blocks that each function below mimics. + # Step 6.1: https://github.com/numpy/numpy/blob/45bc13e6d922690eea43b9d807d476e0f243f836/numpy/f2py/f2py2e.py#L331 _set_rules(rules_setts) + # Step 6.2: https://github.com/numpy/numpy/blob/main/numpy/f2py/f2py2e.py#L332-L342 _set_crackfortran(crackfortran_setts) + # Step 6.3: 1. https://github.com/numpy/numpy/blob/45bc13e6d922690eea43b9d807d476e0f243f836/numpy/f2py/f2py2e.py#L440 + # 2. https://github.com/numpy/numpy/blob/main/numpy/f2py/f2py2e.py#L247-L248 _set_capi_maps(capi_maps_setts) + # Step 6.4: 1. https://github.com/numpy/numpy/blob/45bc13e6d922690eea43b9d807d476e0f243f836/numpy/f2py/f2py2e.py#L439 + # 2. https://github.com/numpy/numpy/blob/main/numpy/f2py/f2py2e.py#L471-L473 _set_auxfuncs(auxfuncs_setts) def generate_files(files: List[str], module_name: str, sign_file: Path) -> None: + # Step 8.1: Generate postlist from crackfortran postlist = _callcrackfortran(files, module_name) + + # Step 8.2: Check postlist. This function is taken from the following code: + # https://github.com/numpy/numpy/blob/main/numpy/f2py/f2py2e.py#L443-L456 _check_postlist(postlist, sign_file) if(sign_file): + # Step 8.3: Generate signature file, take from this code piece + # https://github.com/numpy/numpy/blob/main/numpy/f2py/f2py2e.py#L343-L350 _generate_signature(postlist, sign_file) return if(module_name): + # Step 8.4: Same as the buildmodules folder of f2py2e _buildmodules(postlist) def compile_dist(ext_args: Dict[str, Any], link_resources: List[str], build_dir: Path, fc_flags: List[str], flib_flags: List[str], quiet_build: bool) -> None: + # Step 7.2: The entire code below mimics 'f2py2e:run_compile()' + # https://github.com/numpy/numpy/blob/main/numpy/f2py/f2py2e.py#L647-L669 _set_dependencies_dist(ext_args, link_resources) f2py_dir = get_f2py_dir() ext = Extension(**ext_args) @@ -283,6 +299,10 @@ def compile_dist(ext_args: Dict[str, Any], link_resources: List[str], build_dir: f2py_build_flags.extend(['config_fc'] + fc_flags) if flib_flags: f2py_build_flags.extend(['build_ext'] + flib_flags) + + # f2py2e used to pass `script_name` and `script_args` through `sys.argv` array + # Now we are passing it as attributes. They will be read later distutils core + # https://github.com/pypa/distutils/blob/main/distutils/core.py#L131-L134 setup(ext_modules=[ext], script_name=f2py_dir, script_args=f2py_build_flags) def segregate_files(files: List[str]) -> Tuple[List[str], List[str], List[str], List[str], List[str]]: From d8061f4f071f2d67d7288fdef4abe1cf223c6ac2 Mon Sep 17 00:00:00 2001 From: Namami Shanker Date: Mon, 25 Jul 2022 10:17:31 +0530 Subject: [PATCH 098/132] ENH: Use builtin types in place of typing --- numpy/f2py/f2pyarg.py | 19 ++++++++++--------- numpy/f2py/service.py | 40 +++++++++++++++++++++------------------- numpy/f2py/utils.py | 6 ++++-- 3 files changed, 35 insertions(+), 30 deletions(-) diff --git a/numpy/f2py/f2pyarg.py b/numpy/f2py/f2pyarg.py index e199805c2cc4..386581424558 100644 --- a/numpy/f2py/f2pyarg.py +++ b/numpy/f2py/f2pyarg.py @@ -15,12 +15,13 @@ implementation in terms of subparsers """ +from __future__ import annotations + import argparse import logging import pathlib from numpy.version import version as __version__ -from typing import List, Dict, Tuple, Any from .service import check_dccomp, check_npfcomp, check_dir, generate_files, segregate_files, get_f2py_modulename, wrapper_settings, compile_dist from .utils import open_build_dir @@ -214,7 +215,7 @@ def __call__(self, parser, namespace, values, option_string=None): parser.add_argument( "Fortran Files", metavar="", - action="extend", # List storage + action="extend", # list storage nargs="*", help="""Paths to fortran/signature files that will be scanned for in order to determine their signatures.""", @@ -594,10 +595,10 @@ def __call__(self, parser, namespace, values, option_string=None): # Main Process # ################ -def get_additional_headers(rem: List[str]) -> List[str]: +def get_additional_headers(rem: list[str]) -> list[str]: return [val[8:] for val in rem if val[:8] == '-include'] -def get_f2pyflags_dist(args: argparse.Namespace, skip_funcs: List[str], only_funcs: List[str]) -> List[str]: +def get_f2pyflags_dist(args: argparse.Namespace, skip_funcs: list[str], only_funcs: list[str]) -> list[str]: # Distutils requires 'f2py_options' which will be a subset of # sys.argv array received. This function reconstructs the array # from received args. @@ -625,7 +626,7 @@ def get_f2pyflags_dist(args: argparse.Namespace, skip_funcs: List[str], only_fun f2py_flags.extend(['--f2cmap', str(args.f2cmap)]) return f2py_flags -def get_fortran_library_flags(args: argparse.Namespace) -> List[str]: +def get_fortran_library_flags(args: argparse.Namespace) -> list[str]: flib_flags = [] if args.fcompiler: flib_flags.append(f'--fcompiler={args.fcompiler[0]}') @@ -633,7 +634,7 @@ def get_fortran_library_flags(args: argparse.Namespace) -> List[str]: flib_flags.append(f'--compiler={args.compiler[0]}') return flib_flags -def get_fortran_compiler_flags(args: argparse.Namespace) -> List[str]: +def get_fortran_compiler_flags(args: argparse.Namespace) -> list[str]: fc_flags = [] if(args.help_fcompiler): fc_flags.append('--help-fcompiler') @@ -657,7 +658,7 @@ def get_fortran_compiler_flags(args: argparse.Namespace) -> List[str]: fc_flags.append('--debug') -def get_module_name(args: argparse.Namespace, pyf_files: List[str]) -> str: +def get_module_name(args: argparse.Namespace, pyf_files: list[str]) -> str: if(args.module is not None): return args.module[0] if args.c: @@ -676,7 +677,7 @@ def get_signature_file(args: argparse.Namespace, build_dir: pathlib.Path) -> pat parser.exit() return sign_file -def segregate_posn_args(args: argparse.Namespace) -> Tuple[List[str], List[str], List[str]]: +def segregate_posn_args(args: argparse.Namespace) -> tuple[list[str], list[str], list[str]]: # Currently, argparse does not recognise 'skip:' and 'only:' as optional args # and clubs them all in "Fortran Files" attr. This function segregates them. funcs = {"skip:": [], "only:": []} @@ -693,7 +694,7 @@ def segregate_posn_args(args: argparse.Namespace) -> Tuple[List[str], List[str], funcs[mode].append(arg) return files, funcs['skip:'], funcs['only:'] -def process_args(args: argparse.Namespace, rem: List[str]) -> None: +def process_args(args: argparse.Namespace, rem: list[str]) -> None: if args.help: parser.print_help() parser.exit() diff --git a/numpy/f2py/service.py b/numpy/f2py/service.py index 6885b58f379d..98b4be8fd8d8 100644 --- a/numpy/f2py/service.py +++ b/numpy/f2py/service.py @@ -1,9 +1,11 @@ +from __future__ import annotations + import sys import logging import re from pathlib import Path, PurePath -from typing import List, Dict, Tuple, Any, Optional +from typing import Any, Optional # Distutil dependencies from numpy.distutils.misc_util import dict_append @@ -123,12 +125,12 @@ def check_npfcomp(opt: str) -> str: raise RuntimeError(f"{opt} is not an np.distutils supported compiler, choose from {fchoices}") -def _set_additional_headers(headers: List[str]) -> None: +def _set_additional_headers(headers: list[str]) -> None: for header in headers: cfuncs.outneeds['userincludes'].append(header[1:-1]) cfuncs.userincludes[header[1:-1]] = f"#include {header}" -def _set_crackfortran(crackfortran_setts: Dict[str, Any]) -> None: +def _set_crackfortran(crackfortran_setts: dict[str, Any]) -> None: crackfortran.reset_global_f2py_vars() crackfortran.f77modulename = crackfortran_setts["module"] crackfortran.include_paths[:] = crackfortran_setts["include_paths"] @@ -138,19 +140,19 @@ def _set_crackfortran(crackfortran_setts: Dict[str, Any]) -> None: crackfortran.onlyfuncs = crackfortran_setts["onlyfuncs"] crackfortran.dolowercase = crackfortran_setts["do-lower"] -def _set_rules(rules_setts: Dict[str, Any]) -> None: +def _set_rules(rules_setts: dict[str, Any]) -> None: rules.options = rules_setts -def _set_capi_maps(capi_maps_setts: Dict[str, Any]) -> None: +def _set_capi_maps(capi_maps_setts: dict[str, Any]) -> None: capi_maps.load_f2cmap_file(capi_maps_setts["f2cmap"]) _set_additional_headers(capi_maps_setts["headers"]) -def _set_auxfuncs(aux_funcs_setts: Dict[str, Any]) -> None: +def _set_auxfuncs(aux_funcs_setts: dict[str, Any]) -> None: auxfuncs.options = {'verbose': aux_funcs_setts['verbose']} auxfuncs.debugoptions = aux_funcs_setts["debug"] auxfuncs.wrapfuncs = aux_funcs_setts['wrapfuncs'] -def _dict_append(d_out: Dict[str, Any], d_in: Dict[str, Any]) -> None: +def _dict_append(d_out: dict[str, Any], d_in: dict[str, Any]) -> None: for (k, v) in d_in.items(): if k not in d_out: d_out[k] = [] @@ -159,11 +161,11 @@ def _dict_append(d_out: Dict[str, Any], d_in: Dict[str, Any]) -> None: else: d_out[k].append(v) -def _buildmodules(lst: List[Dict[str, Any]]) -> Dict[str, Any]: +def _buildmodules(lst: list[dict[str, Any]]) -> dict[str, Any]: cfuncs.buildcfuncs() outmess('Building modules...\n') modules, mnames = [], [] - isusedby: Dict[str, List[Any]] = {} + isusedby: dict[str, list[Any]] = {} for item in lst: if '__user__' in item['name']: cb_rules.buildcallbacks(item) @@ -175,7 +177,7 @@ def _buildmodules(lst: List[Dict[str, Any]]) -> Dict[str, Any]: isusedby[u].append(item['name']) modules.append(item) mnames.append(item['name']) - ret: Dict[str, Any] = {} + ret: dict[str, Any] = {} for module, name in zip(modules, mnames): if name in isusedby: outmess('\tSkipping module "%s" which is used by %s.\n' % ( @@ -195,7 +197,7 @@ def _buildmodules(lst: List[Dict[str, Any]]) -> Dict[str, Any]: return ret -def _generate_signature(postlist: List[Dict[str, Any]], sign_file: Path) -> None: +def _generate_signature(postlist: list[dict[str, Any]], sign_file: Path) -> None: outmess(f"Saving signatures to file {sign_file}" + "\n") pyf = crackfortran.crack2fortran(postlist) if sign_file in {"-", "stdout"}: @@ -204,8 +206,8 @@ def _generate_signature(postlist: List[Dict[str, Any]], sign_file: Path) -> None with open(sign_file, "w") as f: f.write(pyf) -def _check_postlist(postlist: List[Dict[str, Any]], sign_file: Path) -> None: - isusedby: Dict[str, List[Any]] = {} +def _check_postlist(postlist: list[dict[str, Any]], sign_file: Path) -> None: + isusedby: dict[str, list[Any]] = {} for plist in postlist: if 'use' in plist: for u in plist['use'].keys(): @@ -229,14 +231,14 @@ def _check_postlist(postlist: List[Dict[str, Any]], sign_file: Path) -> None: outmess( 'Tip: If your original code is Fortran source then you must use -m option.\n') -def _callcrackfortran(files: List[str], module_name: str) -> List[Dict[str, Any]]: +def _callcrackfortran(files: list[str], module_name: str) -> list[dict[str, Any]]: postlist = crackfortran.crackfortran([str(file) for file in files]) for mod in postlist: mod["coutput"] = f"{module_name}module.c" mod["f2py_wrapper_output"] = f"{module_name}-f2pywrappers.f" return postlist -def _set_dependencies_dist(ext_args: Dict[str, Any], link_resource: List[str]) -> None: +def _set_dependencies_dist(ext_args: dict[str, Any], link_resource: list[str]) -> None: for dep in link_resource: info = get_info(dep) if not info: @@ -255,7 +257,7 @@ def get_f2py_modulename(source: str) -> Optional[str]: break return name -def wrapper_settings(rules_setts: Dict[str, Any], crackfortran_setts: Dict[str, Any], capi_maps_setts: Dict[str, Any], auxfuncs_setts: Dict[str, Any]) -> None: +def wrapper_settings(rules_setts: dict[str, Any], crackfortran_setts: dict[str, Any], capi_maps_setts: dict[str, Any], auxfuncs_setts: dict[str, Any]) -> None: # This function also mimics f2py2e. I have added the link to specific code blocks that each function below mimics. # Step 6.1: https://github.com/numpy/numpy/blob/45bc13e6d922690eea43b9d807d476e0f243f836/numpy/f2py/f2py2e.py#L331 _set_rules(rules_setts) @@ -268,7 +270,7 @@ def wrapper_settings(rules_setts: Dict[str, Any], crackfortran_setts: Dict[str, # 2. https://github.com/numpy/numpy/blob/main/numpy/f2py/f2py2e.py#L471-L473 _set_auxfuncs(auxfuncs_setts) -def generate_files(files: List[str], module_name: str, sign_file: Path) -> None: +def generate_files(files: list[str], module_name: str, sign_file: Path) -> None: # Step 8.1: Generate postlist from crackfortran postlist = _callcrackfortran(files, module_name) @@ -284,7 +286,7 @@ def generate_files(files: List[str], module_name: str, sign_file: Path) -> None: # Step 8.4: Same as the buildmodules folder of f2py2e _buildmodules(postlist) -def compile_dist(ext_args: Dict[str, Any], link_resources: List[str], build_dir: Path, fc_flags: List[str], flib_flags: List[str], quiet_build: bool) -> None: +def compile_dist(ext_args: dict[str, Any], link_resources: list[str], build_dir: Path, fc_flags: list[str], flib_flags: list[str], quiet_build: bool) -> None: # Step 7.2: The entire code below mimics 'f2py2e:run_compile()' # https://github.com/numpy/numpy/blob/main/numpy/f2py/f2py2e.py#L647-L669 _set_dependencies_dist(ext_args, link_resources) @@ -305,7 +307,7 @@ def compile_dist(ext_args: Dict[str, Any], link_resources: List[str], build_dir: # https://github.com/pypa/distutils/blob/main/distutils/core.py#L131-L134 setup(ext_modules=[ext], script_name=f2py_dir, script_args=f2py_build_flags) -def segregate_files(files: List[str]) -> Tuple[List[str], List[str], List[str], List[str], List[str]]: +def segregate_files(files: list[str]) -> tuple[list[str], list[str], list[str], list[str], list[str]]: """ Segregate files into five groups: * Fortran 77 files diff --git a/numpy/f2py/utils.py b/numpy/f2py/utils.py index 148f75467ea0..1fe95a81dde9 100644 --- a/numpy/f2py/utils.py +++ b/numpy/f2py/utils.py @@ -1,10 +1,12 @@ """Global f2py utilities.""" +from __future__ import annotations + import contextlib import tempfile import shutil -from typing import Optional, List +from typing import Optional from pathlib import Path def get_f2py_dir() -> Path: @@ -12,7 +14,7 @@ def get_f2py_dir() -> Path: return Path(__file__).resolve().parent @contextlib.contextmanager -def open_build_dir(build_dir: Optional[List[str]], compile: bool) -> Path: +def open_build_dir(build_dir: Optional[list[str]], compile: bool) -> Path: """Create build directory if the user specifies it, Otherwise, create a temporary directory and remove it. From 3aa40b722d90b6528fd0a5a62a6b6b350970d3a7 Mon Sep 17 00:00:00 2001 From: Namami Shanker Date: Fri, 5 Aug 2022 15:49:42 +0530 Subject: [PATCH 099/132] ENH: Return c wrapper path --- numpy/f2py/service.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/numpy/f2py/service.py b/numpy/f2py/service.py index 98b4be8fd8d8..4d18342fbc27 100644 --- a/numpy/f2py/service.py +++ b/numpy/f2py/service.py @@ -257,7 +257,7 @@ def get_f2py_modulename(source: str) -> Optional[str]: break return name -def wrapper_settings(rules_setts: dict[str, Any], crackfortran_setts: dict[str, Any], capi_maps_setts: dict[str, Any], auxfuncs_setts: dict[str, Any]) -> None: +def wrapper_settings(rules_setts: dict[str, Any], crackfortran_setts: dict[str, Any], capi_maps_setts: dict[str, Any], auxfuncs_setts: dict[str, Any]) -> Optional[Path]: # This function also mimics f2py2e. I have added the link to specific code blocks that each function below mimics. # Step 6.1: https://github.com/numpy/numpy/blob/45bc13e6d922690eea43b9d807d476e0f243f836/numpy/f2py/f2py2e.py#L331 _set_rules(rules_setts) @@ -284,7 +284,9 @@ def generate_files(files: list[str], module_name: str, sign_file: Path) -> None: return if(module_name): # Step 8.4: Same as the buildmodules folder of f2py2e - _buildmodules(postlist) + ret = _buildmodules(postlist) + c_wrapper = ret.get(module_name).get('csrc')[0] + return Path(c_wrapper) def compile_dist(ext_args: dict[str, Any], link_resources: list[str], build_dir: Path, fc_flags: list[str], flib_flags: list[str], quiet_build: bool) -> None: # Step 7.2: The entire code below mimics 'f2py2e:run_compile()' From ac164c7514eb05961a5fe9f7da08bd7746cda10d Mon Sep 17 00:00:00 2001 From: Namami Shanker Date: Sun, 7 Aug 2022 20:12:48 +0530 Subject: [PATCH 100/132] ENH: Add --backend flag for meson integration --- numpy/f2py/f2pyarg.py | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/numpy/f2py/f2pyarg.py b/numpy/f2py/f2pyarg.py index 386581424558..69641b1fdc49 100644 --- a/numpy/f2py/f2pyarg.py +++ b/numpy/f2py/f2pyarg.py @@ -20,6 +20,7 @@ import argparse import logging import pathlib +import enum from numpy.version import version as __version__ @@ -158,6 +159,36 @@ def __call__(self, parser, namespace, values, option_string=None): items.append((value, None)) setattr(namespace, self.dest, items) +class EnumAction(argparse.Action): + """ + Argparse action for handling Enums + """ + def __init__(self, **kwargs): + # Pop off the type value + enum_type = kwargs.pop("type", None) + + # Ensure an Enum subclass is provided + if enum_type is None: + raise ValueError("type must be assigned an Enum when using EnumAction") + if not issubclass(enum_type, enum.Enum): + raise TypeError("type must be an Enum when using EnumAction") + + # Generate choices from the Enum + kwargs.setdefault("choices", tuple(e.value for e in enum_type)) + + super(EnumAction, self).__init__(**kwargs) + + self._enum = enum_type + + def __call__(self, parser, namespace, values, option_string=None): + # Convert value back into an Enum + value = self._enum(values) + setattr(namespace, self.dest, value) + + +class Backends(enum.Enum): + Meson = "meson" + Distutils = "distutils" ########## # Parser # @@ -583,6 +614,10 @@ def __call__(self, parser, namespace, values, option_string=None): help="The link helpers for numpy distutils" ) +parser.add_argument('--backend', + type=Backends, + default="distutils", + action=EnumAction) # The rest, only works for files, since we expect: # .o .so .a From 58fb9c38545a42931a09569457e23473dcb83115 Mon Sep 17 00:00:00 2001 From: Namami Shanker Date: Sun, 7 Aug 2022 20:15:26 +0530 Subject: [PATCH 101/132] BUG: Seperate paths by colon --- numpy/f2py/f2pyarg.py | 26 ++++++++++++++++++++++---- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/numpy/f2py/f2pyarg.py b/numpy/f2py/f2pyarg.py index 69641b1fdc49..858634f64ae3 100644 --- a/numpy/f2py/f2pyarg.py +++ b/numpy/f2py/f2pyarg.py @@ -19,6 +19,7 @@ import argparse import logging +import os import pathlib import enum @@ -185,6 +186,22 @@ def __call__(self, parser, namespace, values, option_string=None): value = self._enum(values) setattr(namespace, self.dest, value) +class IncludePathAction(argparse.Action): + """Custom action to extend paths when --include-paths : is called""" + def __init__(self, option_strings, dest, nargs="?", **kwargs): + """Initialization of the flag flag + + Mimics the parent + """ + super(IncludePathAction, self).__init__(option_strings, dest, nargs="?", **kwargs) + + def __call__(self, parser, namespace, values, option_string=None): + """Split the paths by ':' convert them to path and append them to the attribute""" + items = getattr(namespace, self.dest) or [] + if values: + items.extend([pathlib.Path(path) for path in values.split(os.pathsep)]) + setattr(namespace, self.dest, items) + class Backends(enum.Enum): Meson = "meson" @@ -367,11 +384,12 @@ class Backends(enum.Enum): parser.add_argument( "--include-paths", + nargs='?', + dest="include_paths", + action=IncludePathAction, metavar=":", - action="extend", + type=str, default=[], - nargs="*", - type=pathlib.Path, help="Search include files from the given directories.", ) @@ -656,7 +674,7 @@ def get_f2pyflags_dist(args: argparse.Namespace, skip_funcs: list[str], only_fun if(only_funcs): f2py_flags.extend(['only:']+only_funcs + [':']) if(args.include_paths): - f2py_flags.extend(['--include-paths']+args.include_paths) + f2py_flags.extend(['--include-paths']+[str(include_path) for include_path in args.include_paths]) if(args.f2cmap): f2py_flags.extend(['--f2cmap', str(args.f2cmap)]) return f2py_flags From e4c5d0bd0c7e546baa2ab2cbb9994edc2bf6fec1 Mon Sep 17 00:00:00 2001 From: Namami Shanker Date: Sun, 7 Aug 2022 20:16:27 +0530 Subject: [PATCH 102/132] BUG: Add default values and pass paths as strings --- numpy/f2py/f2pyarg.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/numpy/f2py/f2pyarg.py b/numpy/f2py/f2pyarg.py index 858634f64ae3..8d23dc5085ec 100644 --- a/numpy/f2py/f2pyarg.py +++ b/numpy/f2py/f2pyarg.py @@ -554,6 +554,7 @@ class Backends(enum.Enum): metavar="/path/to/lib/", nargs="*", action="extend", + default=[], help="Path to library" ) @@ -592,6 +593,7 @@ class Backends(enum.Enum): type=pathlib.Path, metavar="/path/to/include", nargs="*", + default=[], action="extend", help="Include directories" ) @@ -833,8 +835,8 @@ def process_args(args: argparse.Namespace, rem: list[str]) -> None: ext_args = { 'name': module_name, 'sources': pyf_files + f77_files + f90_files, - 'include_dirs': args.include_dirs, - 'library_dirs': args.library_path, + 'include_dirs': [include_dir.absolute() for include_dir in args.include_dirs], + 'library_dirs': [lib_path.absolute() for lib_path in args.library_path], 'libraries': args.library_name, 'define_macros': args.define_macros, 'undef_macros': args.undef_macros, From 5f599aa3f03ad6226f9b737850c5b2710c83d869 Mon Sep 17 00:00:00 2001 From: Namami Shanker Date: Sun, 7 Aug 2022 20:18:51 +0530 Subject: [PATCH 103/132] ENH: Add initial meson plugin --- numpy/f2py/backends/__init__.py | 6 ++ numpy/f2py/backends/backend.py | 109 +++++++++++++++++++ numpy/f2py/backends/meson_backend.py | 137 ++++++++++++++++++++++++ numpy/f2py/backends/src/meson.build.src | 19 ++++ 4 files changed, 271 insertions(+) create mode 100644 numpy/f2py/backends/__init__.py create mode 100644 numpy/f2py/backends/backend.py create mode 100644 numpy/f2py/backends/meson_backend.py create mode 100644 numpy/f2py/backends/src/meson.build.src diff --git a/numpy/f2py/backends/__init__.py b/numpy/f2py/backends/__init__.py new file mode 100644 index 000000000000..4ef3593b33c9 --- /dev/null +++ b/numpy/f2py/backends/__init__.py @@ -0,0 +1,6 @@ +from .backend import Backend +from .meson_backend import MesonBackend + +backends = { + 'meson': MesonBackend +} \ No newline at end of file diff --git a/numpy/f2py/backends/backend.py b/numpy/f2py/backends/backend.py new file mode 100644 index 000000000000..8c37d2bb2360 --- /dev/null +++ b/numpy/f2py/backends/backend.py @@ -0,0 +1,109 @@ +from __future__ import annotations + +from pathlib import Path +from abc import ABC, abstractmethod + +import numpy +import numpy.f2py as f2py + +class Backend(ABC): + """ + Superclass for backend compilation plugins to extend. + + Parameters + ---------- + + """ + + def __init__(self, module_name, fortran_compiler: str, c_compiler: str, + f77exec: Path, f90exec: Path, f77_flags: list[str], + f90_flags: list[str], include_path: list[Path], + include_dirs: list[Path], external_resources: list[str], + linker_dirs: list[Path], linker_libname: list[str], + macros: list[tuple[str, str]], unset_macros: list[str], + debug: bool, opt_flags: list[str], arch_flags: list[str], + no_opt: bool, no_arch: bool) -> None: + """ + The class is initialized with f2py compile options. + The parameters are mappings of f2py compilation flags. + + Parameters + ---------- + module_name : str + The name of the module to be compiled. (-m) + fortran_compiler : str + Name of the Fortran compiler to use. (--fcompiler) + c_compiler : str + Name of the C compiler to use. (--ccompiler) + f77exec : Pathlike + Path to the fortran compiler for Fortran 77 files (--f77exec) + f90exec : Pathlike + Path to the fortran compiler for Fortran 90 and above files (--f90exec) + f77_flags : list + List of flags to pass to the fortran compiler for Fortran 77 files (--f77flags) + f90_flags : list + List of flags to pass to the fortran compiler for Fortran 90 and above files (--f90flags) + include_path : list + Search include files from given directories (--include-paths) + include_dirs : list + Append directory to the list of directories searched for include files. (-I) + external_resources : list + Link the extension module with (--link-) + linker_libname : list + Use the library when linking. (-l) + linker_dirs : list + Add directory to the list of directories to be searched for `-l`. (-L) + opt_flags : list + Optimization flags to pass to the compiler. (--opt) + arch_flags : list + Architectire specific flags to pass to the compiler (--arch) + no_opt : bool + Disable optimization. (--no-opt) + no_arch : bool + Disable architecture specific optimizations. (--no-arch) + debug : bool + Enable debugging. (--debug) + + """ + self.module_name = module_name + self.fortran_compiler = fortran_compiler + self.c_compiler = c_compiler + self.f77exec = f77exec + self.f90exec = f90exec + self.f77_flags = f77_flags + self.f90_flags = f90_flags + self.include_path = include_path + self.include_dirs = include_dirs + self.external_resources = external_resources + self.linker_dirs = linker_dirs + self.linker_libname = linker_libname + self.macros = macros + self.unset_macros = unset_macros + self.debug = debug + self.opt_flags = opt_flags + self.arch_flags = arch_flags + self.no_opt = no_opt + self.no_arch = no_arch + + def numpy_install_path(self) -> Path: + """ + Returns the install path for numpy. + """ + return Path(numpy.__file__).parent + + def numpy_get_include(self) -> Path: + """ + Returns the include paths for numpy. + """ + return Path(numpy.get_include()) + + def f2py_get_include(self) -> Path: + """ + Returns the include paths for f2py. + """ + return Path(f2py.get_include()) + + @abstractmethod + def compile(self, fortran_sources: Path, c_wrapper: Path, build_dir: Path) -> None: + """Compile the wrapper.""" + pass \ No newline at end of file diff --git a/numpy/f2py/backends/meson_backend.py b/numpy/f2py/backends/meson_backend.py new file mode 100644 index 000000000000..c199d07d17c6 --- /dev/null +++ b/numpy/f2py/backends/meson_backend.py @@ -0,0 +1,137 @@ +import os +import errno +import subprocess +from pathlib import Path + +from .backend import Backend +from string import Template + +class MesonTemplate: + """Template meson build file generation class.""" + def __init__(self, module_name: str, numpy_install_path, numpy_get_include: Path, f2py_get_include: Path, c_wrapper: Path, fortran_sources: list[Path], dependencies: list[str], include_path: list[Path]): + self.module_name = module_name + self.numpy_install_path = numpy_install_path + self.build_template_path = numpy_install_path / "f2py" / "backends" / "src" / "meson.build.src" + self.sources = fortran_sources + self.numpy_get_include = numpy_get_include + self.f2py_get_include = f2py_get_include + self.c_wrapper = c_wrapper + self.dependencies = dependencies + self.include_directories = include_path + self.substitutions = {} + self.pipeline = [self.initialize_template, + self.sources_substitution, + self.dependencies_substitution, + self.include_directories_subtitution] + + @property + def meson_build_template(self) -> str: + if(not self.build_template_path.is_file()): + raise FileNotFoundError(errno.ENOENT, f"Meson build template {self.build_template_path.absolute()} does not exist.") + return self.build_template_path.read_text() + + def initialize_template(self) -> None: + """Initialize with module name and external NumPy and F2PY C libraries.""" + self.substitutions['modulename'] = self.module_name + self.substitutions['numpy_get_include'] = self.numpy_get_include.absolute() + self.substitutions['f2py_get_include'] = self.f2py_get_include.absolute() + + def sources_substitution(self) -> None: + self.substitutions["source_list"] = ",".join(["\'"+str(source.absolute())+"\'" for source in self.sources]) + self.substitutions["c_wrapper"] = str(self.c_wrapper.absolute()) if self.c_wrapper else "" + + def dependencies_substitution(self) -> None: + self.substitutions["dependencies_list"] = ", ".join([f"dependency('{dependecy}')" for dependecy in self.dependencies]) + + def include_directories_subtitution(self) -> None: + self.substitutions["include_directories_list"] = ", ".join([f"include_directories('{include_directory}')" for include_directory in self.include_directories]) + + def generate_meson_build(self) -> str: + for node in self.pipeline: + node() + template = Template(self.meson_build_template) + return template.substitute(self.substitutions) + + +class MesonBackend(Backend): + + def __init__(self, module_name: str = 'untitled', fortran_compiler: str = None, c_compiler: str = None, f77exec: Path = None, f90exec: Path = None, f77_flags: list[str] = None, f90_flags: list[str] = None, include_path: list[Path] = None, include_dirs: list[Path] = None, external_resources: list[str] = None, linker_dirs: list[Path] = None, linker_libname: list[str] = None, macros: list[tuple[str, str]] = None, unset_macros: list[str] = None, debug: bool = False, opt_flags: list[str] = None, arch_flags: list[str] = None, no_opt: bool = False, no_arch: bool = False) -> None: + if f77_flags is None: + f77_flags = [] + if include_path is None: + include_path = [] + if include_dirs is None: + include_dirs = [] + if external_resources is None: + external_resources = [] + if linker_dirs is None: + linker_dirs = [] + if linker_libname is None: + linker_libname = [] + if macros is None: + macros = [] + if unset_macros is None: + unset_macros = [] + if opt_flags is None: + opt_flags = [] + if arch_flags is None: + arch_flags = [] + super().__init__(module_name, fortran_compiler, c_compiler, f77exec, f90exec, f77_flags, f90_flags, include_path, include_dirs, external_resources, linker_dirs, linker_libname, macros, unset_macros, debug, opt_flags, arch_flags, no_opt, no_arch) + + self.c_wrapper: Path = None + self.fortran_sources: list[Path] = [] + self.template = Template(self.fortran_sources) + + def _get_optimization_level(self): + if self.no_arch and not self.no_opt : + return 2 + elif self.no_opt : + return 0 + return 3 + + def _set_environment_variables(self) -> None: + if self.fortran_compiler: + os.putenv("FC", self.fortran_compiler) + if self.c_compiler: + os.putenv("CC", self.c_compiler) + + def _get_build_command(self): + return ["meson", "setup", "builddir", "-Ddebug=true" if self.debug else "-Ddebug=false", f"-Doptimization={str(self._get_optimization_level())}"] + + def load_wrapper(self, wrapper_path: Path) -> None: + wrapper_path: Path = Path(wrapper_path) + if not wrapper_path.is_file(): + raise FileNotFoundError(errno.ENOENT, f"{wrapper_path.absolute()} does not exist.") + self.c_wrapper = wrapper_path + + def load_sources(self, fortran_sources: list[Path]) -> None: + for fortran_source in fortran_sources: + fortran_source = Path(fortran_source) + if not fortran_source.is_file(): + raise FileNotFoundError(errno.ENOENT, f"{fortran_source.absolute()} does not exist.") + self.fortran_sources.append(fortran_source) + + def write_meson_build(self, build_dir: Path) -> None: + """Writes the meson build file at specified location""" + meson_template = MesonTemplate(self.module_name, super().numpy_install_path(), self.numpy_get_include(), self.f2py_get_include(), self.c_wrapper, self.fortran_sources, self.external_resources, self.include_path+self.include_dirs) + src = meson_template.generate_meson_build() + meson_build_file = build_dir / "meson.build" + meson_build_file.write_text(src) + return meson_build_file + + def run_meson(self, build_dir: Path): + self._set_environment_variables() + completed_process = subprocess.run(self._get_build_command(), cwd=build_dir) + if(completed_process.returncode != 0): + raise subprocess.CalledProcessError(completed_process.returncode, completed_process.args) + completed_process = subprocess.run(["meson", "compile", "-C", "builddir"], cwd=build_dir) + if(completed_process.returncode != 0): + raise subprocess.CalledProcessError(completed_process.returncode, completed_process.args) + + + def compile(self, fortran_sources: list[Path], c_wrapper: Path, build_dir: Path) -> None: + self.load_wrapper(c_wrapper) + self.load_sources(fortran_sources) + self.write_meson_build(build_dir) + self.write_meson_build(build_dir) + self.run_meson(build_dir) diff --git a/numpy/f2py/backends/src/meson.build.src b/numpy/f2py/backends/src/meson.build.src new file mode 100644 index 000000000000..476e0d679303 --- /dev/null +++ b/numpy/f2py/backends/src/meson.build.src @@ -0,0 +1,19 @@ +project('${modulename}', 'c', + version : '0.1', + default_options : ['warning_level=2']) + +add_languages('fortran') + +py_mod = import('python') +py3 = py_mod.find_installation('python3') +py3_dep = py3.dependency() +message(py3.path()) +message(py3.get_install_dir()) + +incdir_numpy = '${numpy_get_include}' + +incdir_f2py = '${f2py_get_include}' + +inc_np = include_directories(incdir_numpy, incdir_f2py) + +py3.extension_module('${modulename}', '${c_wrapper}', ${source_list}, incdir_f2py+'/fortranobject.c', include_directories: [inc_np, ${include_directories_list}], dependencies : [py3_dep, ${dependencies_list}], install : true) \ No newline at end of file From e9fe1ce5dd49befe6f25e363b8ca06bcb184fe4d Mon Sep 17 00:00:00 2001 From: Namami Shanker Date: Sun, 7 Aug 2022 20:19:20 +0530 Subject: [PATCH 104/132] ENH: Use meson backend in f2pyarg --- numpy/f2py/f2pyarg.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/numpy/f2py/f2pyarg.py b/numpy/f2py/f2pyarg.py index 8d23dc5085ec..22064ea0cf17 100644 --- a/numpy/f2py/f2pyarg.py +++ b/numpy/f2py/f2pyarg.py @@ -28,6 +28,7 @@ from .service import check_dccomp, check_npfcomp, check_dir, generate_files, segregate_files, get_f2py_modulename, wrapper_settings, compile_dist from .utils import open_build_dir from .auxfuncs import outmess +from .backends import backends, Backend ################## # Temp Variables # @@ -821,7 +822,7 @@ def process_args(args: argparse.Namespace, rem: list[str]) -> None: # Step 7: If user has asked for compilation. Mimic 'run_compile' from f2py2e # Disutils receives all the options and builds the extension. - if(args.c): + if(args.c and args.backend == Backends.Distutils.value): link_resource = args.link_resource # The 3 functions below generate arrays of flag similar to how @@ -846,7 +847,10 @@ def process_args(args: argparse.Namespace, rem: list[str]) -> None: compile_dist(ext_args, link_resource, build_dir, fc_flags, flib_flags, args.quiet) else: # Step 8: Generate wrapper or signature file if compile flag is not given - generate_files(f77_files + f90_files, module_name, sign_file) + c_wrapper = generate_files(f77_files + f90_files, module_name, sign_file) + if c_wrapper and args.c: + backend: Backend = backends.get(args.backend.value)(module_name=module_name, include_dirs=args.include_dirs, include_path=args.include_paths, external_resources=args.link_resource, debug=args.debug) + backend.compile(f77_files + f90_files, c_wrapper, build_dir) def main(): logger = logging.getLogger("f2py_cli") From 379bb56892af96cb3ac757cf0da0c1cc6f597e0e Mon Sep 17 00:00:00 2001 From: Namami Shanker Date: Sun, 7 Aug 2022 20:21:01 +0530 Subject: [PATCH 105/132] MAINT: Sort by alphabetical order --- numpy/tests/test_public_api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/numpy/tests/test_public_api.py b/numpy/tests/test_public_api.py index 7eea1eb2fdc4..21ccea5a3aa9 100644 --- a/numpy/tests/test_public_api.py +++ b/numpy/tests/test_public_api.py @@ -253,12 +253,12 @@ def test_NPY_NO_EXPORT(): "f2py.crackfortran", "f2py.diagnose", "f2py.f2py2e", + "f2py.f2pyarg", "f2py.f90mod_rules", "f2py.func2subr", "f2py.rules", "f2py.symbolic", "f2py.use_rules", - "f2py.f2pyarg", "f2py.service", "f2py.utils", "fft.helper", From f1d93c0fcede2e46e73d78e5b34f4bd8f2158ad0 Mon Sep 17 00:00:00 2001 From: Namami Shanker Date: Fri, 12 Aug 2022 21:43:07 +0530 Subject: [PATCH 106/132] MAINT: Fix typo --- numpy/f2py/f2pyarg.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/numpy/f2py/f2pyarg.py b/numpy/f2py/f2pyarg.py index 22064ea0cf17..5899d5c006f1 100644 --- a/numpy/f2py/f2pyarg.py +++ b/numpy/f2py/f2pyarg.py @@ -190,7 +190,7 @@ def __call__(self, parser, namespace, values, option_string=None): class IncludePathAction(argparse.Action): """Custom action to extend paths when --include-paths : is called""" def __init__(self, option_strings, dest, nargs="?", **kwargs): - """Initialization of the flag flag + """Initialization of the --include-paths flag Mimics the parent """ From 8c0858b64d726b1cd7a920cd6fdeb33645bbafcb Mon Sep 17 00:00:00 2001 From: Namami Shanker Date: Fri, 12 Aug 2022 21:44:34 +0530 Subject: [PATCH 107/132] ENH: Add custom string flags parsing --- numpy/f2py/f2pyarg.py | 33 ++++++++++++++++++++++++--------- 1 file changed, 24 insertions(+), 9 deletions(-) diff --git a/numpy/f2py/f2pyarg.py b/numpy/f2py/f2pyarg.py index 5899d5c006f1..190a6f402fb6 100644 --- a/numpy/f2py/f2pyarg.py +++ b/numpy/f2py/f2pyarg.py @@ -203,6 +203,20 @@ def __call__(self, parser, namespace, values, option_string=None): items.extend([pathlib.Path(path) for path in values.split(os.pathsep)]) setattr(namespace, self.dest, items) +class ParseStringFlags(argparse.Action): + """Custom action to parse and store flags passed as string + Ex- + f2py --opt="-DDEBUG=1 -O" will be stored as ["-DDEBUG=1", -O]""" + + def __init__(self, option_strings, dest, nargs="1", **kwargs): + """Initialization of the flag, mimics the parent""" + super(ParseStringFlags, self).__init__(option_strings, dest, nargs=1, **kwargs) + + def __call__(self, parser, namespace, values, option_string=None): + """The storage action, mimics the parent""" + items = getattr(namespace, self.dest) or [] + items.extend(value.split(' ') for value in values) + setattr(namespace, self.dest, items) class Backends(enum.Enum): Meson = "meson" @@ -500,38 +514,39 @@ class Backends(enum.Enum): build_helpers.add_argument( "--f77flags", - nargs="*", - action="extend", + nargs=1, + action=ParseStringFlags, help="Specify F77 compiler flags" ) build_helpers.add_argument( "--f90flags", - nargs="*", - action="extend", + nargs=1, + action=ParseStringFlags, help="Specify F90 compiler flags" ) build_helpers.add_argument( "--opt", "--optimization_flags", - nargs="*", + nargs=1, type=str, - action="extend", + action=ParseStringFlags, help="Specify optimization flags" ) build_helpers.add_argument( "--arch", "--architecture_optimizations", - nargs="*", + nargs=1, type=str, - action="extend", + action=ParseStringFlags, help="Specify architecture specific optimization flags" ) build_helpers.add_argument( - "--noopt", + """_summary_ + """ "--noopt", action="store_true", help="Compile without optimization" ) From 5b99e87b5eef027a16d15feea7554eba2e2015b6 Mon Sep 17 00:00:00 2001 From: Namami Shanker Date: Thu, 18 Aug 2022 23:37:00 +0530 Subject: [PATCH 108/132] FIX: Flag nargs --- numpy/f2py/f2pyarg.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/numpy/f2py/f2pyarg.py b/numpy/f2py/f2pyarg.py index 190a6f402fb6..57fcc2fed4a2 100644 --- a/numpy/f2py/f2pyarg.py +++ b/numpy/f2py/f2pyarg.py @@ -568,7 +568,7 @@ class Backends(enum.Enum): "--library-path", type=pathlib.Path, metavar="/path/to/lib/", - nargs="*", + nargs=1, action="extend", default=[], help="Path to library" @@ -596,9 +596,9 @@ class Backends(enum.Enum): build_helpers.add_argument( "-l", "--library_name", - type=pathlib.Path, + type=str, metavar="", - nargs="*", + nargs=1, action="extend", help="Library name" ) From af364d0d2682eb700726e99cc65ebf4c4025c9db Mon Sep 17 00:00:00 2001 From: Namami Shanker Date: Thu, 18 Aug 2022 23:37:22 +0530 Subject: [PATCH 109/132] MAINT: Correct docstring --- numpy/f2py/f2pyarg.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/numpy/f2py/f2pyarg.py b/numpy/f2py/f2pyarg.py index 57fcc2fed4a2..a25e1f0093a3 100644 --- a/numpy/f2py/f2pyarg.py +++ b/numpy/f2py/f2pyarg.py @@ -206,7 +206,7 @@ def __call__(self, parser, namespace, values, option_string=None): class ParseStringFlags(argparse.Action): """Custom action to parse and store flags passed as string Ex- - f2py --opt="-DDEBUG=1 -O" will be stored as ["-DDEBUG=1", -O]""" + f2py --opt="-DDEBUG=1 -O" will be stored as ["-DDEBUG=1", "-O"]""" def __init__(self, option_strings, dest, nargs="1", **kwargs): """Initialization of the flag, mimics the parent""" From 2154c47a322aa2fd491ea6fa7bff123cda690f77 Mon Sep 17 00:00:00 2001 From: Namami Shanker Date: Thu, 18 Aug 2022 23:39:04 +0530 Subject: [PATCH 110/132] MAINT:Refactor variable names according to f2pyarg --- numpy/f2py/backends/backend.py | 30 +++++++++++++++++------------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/numpy/f2py/backends/backend.py b/numpy/f2py/backends/backend.py index 8c37d2bb2360..85a8598ad756 100644 --- a/numpy/f2py/backends/backend.py +++ b/numpy/f2py/backends/backend.py @@ -16,13 +16,13 @@ class Backend(ABC): """ def __init__(self, module_name, fortran_compiler: str, c_compiler: str, - f77exec: Path, f90exec: Path, f77_flags: list[str], - f90_flags: list[str], include_path: list[Path], - include_dirs: list[Path], external_resources: list[str], - linker_dirs: list[Path], linker_libname: list[str], - macros: list[tuple[str, str]], unset_macros: list[str], - debug: bool, opt_flags: list[str], arch_flags: list[str], - no_opt: bool, no_arch: bool) -> None: + f77exec: Path, f90exec: Path, f77_flags: list[str], + f90_flags: list[str], include_paths: list[Path], + include_dirs: list[Path], external_resources: list[str], + linker_libpath: list[Path], linker_libname: list[str], + define_macros: list[tuple[str, str]], undef_macros: list[str], + debug: bool, opt_flags: list[str], arch_flags: list[str], + no_opt: bool, no_arch: bool) -> None: """ The class is initialized with f2py compile options. The parameters are mappings of f2py compilation flags. @@ -43,7 +43,7 @@ def __init__(self, module_name, fortran_compiler: str, c_compiler: str, List of flags to pass to the fortran compiler for Fortran 77 files (--f77flags) f90_flags : list List of flags to pass to the fortran compiler for Fortran 90 and above files (--f90flags) - include_path : list + include_paths : list Search include files from given directories (--include-paths) include_dirs : list Append directory to the list of directories searched for include files. (-I) @@ -51,7 +51,11 @@ def __init__(self, module_name, fortran_compiler: str, c_compiler: str, Link the extension module with (--link-) linker_libname : list Use the library when linking. (-l) - linker_dirs : list + define_macros : list + Define to if present else define to true (-D) + undef_macros : list + Undefine (-U) + linker_libpath : list Add directory to the list of directories to be searched for `-l`. (-L) opt_flags : list Optimization flags to pass to the compiler. (--opt) @@ -72,13 +76,13 @@ def __init__(self, module_name, fortran_compiler: str, c_compiler: str, self.f90exec = f90exec self.f77_flags = f77_flags self.f90_flags = f90_flags - self.include_path = include_path + self.include_paths = include_paths self.include_dirs = include_dirs self.external_resources = external_resources - self.linker_dirs = linker_dirs + self.linker_libpath = linker_libpath self.linker_libname = linker_libname - self.macros = macros - self.unset_macros = unset_macros + self.define_macros = define_macros + self.undef_macros = undef_macros self.debug = debug self.opt_flags = opt_flags self.arch_flags = arch_flags From 2803ac5d9b0818418a643569a385673965c22e2e Mon Sep 17 00:00:00 2001 From: Namami Shanker Date: Thu, 18 Aug 2022 23:39:27 +0530 Subject: [PATCH 111/132] MAINT: Fix docstring --- numpy/f2py/backends/backend.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/numpy/f2py/backends/backend.py b/numpy/f2py/backends/backend.py index 85a8598ad756..0608bcfce207 100644 --- a/numpy/f2py/backends/backend.py +++ b/numpy/f2py/backends/backend.py @@ -10,9 +10,6 @@ class Backend(ABC): """ Superclass for backend compilation plugins to extend. - Parameters - ---------- - """ def __init__(self, module_name, fortran_compiler: str, c_compiler: str, From 5a2706fe0c3caea497a74137072581f4120f6b33 Mon Sep 17 00:00:00 2001 From: Namami Shanker Date: Thu, 18 Aug 2022 23:40:28 +0530 Subject: [PATCH 112/132] FIX: Add flags --- numpy/f2py/backends/meson_backend.py | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/numpy/f2py/backends/meson_backend.py b/numpy/f2py/backends/meson_backend.py index c199d07d17c6..b82e0bd2ee9d 100644 --- a/numpy/f2py/backends/meson_backend.py +++ b/numpy/f2py/backends/meson_backend.py @@ -55,28 +55,32 @@ def generate_meson_build(self) -> str: class MesonBackend(Backend): - def __init__(self, module_name: str = 'untitled', fortran_compiler: str = None, c_compiler: str = None, f77exec: Path = None, f90exec: Path = None, f77_flags: list[str] = None, f90_flags: list[str] = None, include_path: list[Path] = None, include_dirs: list[Path] = None, external_resources: list[str] = None, linker_dirs: list[Path] = None, linker_libname: list[str] = None, macros: list[tuple[str, str]] = None, unset_macros: list[str] = None, debug: bool = False, opt_flags: list[str] = None, arch_flags: list[str] = None, no_opt: bool = False, no_arch: bool = False) -> None: + def __init__(self, module_name: str = 'untitled', fortran_compiler: str = None, c_compiler: str = None, f77exec: Path = None, f90exec: Path = None, f77_flags: list[str] = None, f90_flags: list[str] = None, include_paths: list[Path] = None, include_dirs: list[Path] = None, external_resources: list[str] = None, linker_libpath: list[Path] = None, linker_libname: list[str] = None, define_macros: list[tuple[str, str]] = None, undef_macros: list[str] = None, debug: bool = False, opt_flags: list[str] = None, arch_flags: list[str] = None, no_opt: bool = False, no_arch: bool = False) -> None: if f77_flags is None: f77_flags = [] - if include_path is None: - include_path = [] + if include_paths is None: + include_paths = [] if include_dirs is None: include_dirs = [] if external_resources is None: external_resources = [] - if linker_dirs is None: - linker_dirs = [] + if linker_libpath is None: + linker_libpath = [] if linker_libname is None: linker_libname = [] - if macros is None: - macros = [] - if unset_macros is None: - unset_macros = [] + if define_macros is None: + define_macros = [] + if undef_macros is None: + undef_macros = [] + if f77_flags is None: + f77_flags = [] + if f90_flags is None: + f90_flags = [] if opt_flags is None: opt_flags = [] if arch_flags is None: arch_flags = [] - super().__init__(module_name, fortran_compiler, c_compiler, f77exec, f90exec, f77_flags, f90_flags, include_path, include_dirs, external_resources, linker_dirs, linker_libname, macros, unset_macros, debug, opt_flags, arch_flags, no_opt, no_arch) + super().__init__(module_name, fortran_compiler, c_compiler, f77exec, f90exec, f77_flags, f90_flags, include_paths, include_dirs, external_resources, linker_libpath, linker_libname, define_macros, undef_macros, debug, opt_flags, arch_flags, no_opt, no_arch) self.c_wrapper: Path = None self.fortran_sources: list[Path] = [] From 198895cb66ade5f134968d91290add1658db703f Mon Sep 17 00:00:00 2001 From: Namami Shanker Date: Sat, 20 Aug 2022 11:52:30 +0530 Subject: [PATCH 113/132] ENH: Update f2pyarg with full meson backend params --- numpy/f2py/f2pyarg.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/numpy/f2py/f2pyarg.py b/numpy/f2py/f2pyarg.py index a25e1f0093a3..9fc0183bee0e 100644 --- a/numpy/f2py/f2pyarg.py +++ b/numpy/f2py/f2pyarg.py @@ -864,7 +864,7 @@ def process_args(args: argparse.Namespace, rem: list[str]) -> None: # Step 8: Generate wrapper or signature file if compile flag is not given c_wrapper = generate_files(f77_files + f90_files, module_name, sign_file) if c_wrapper and args.c: - backend: Backend = backends.get(args.backend.value)(module_name=module_name, include_dirs=args.include_dirs, include_path=args.include_paths, external_resources=args.link_resource, debug=args.debug) + backend: Backend = backends.get(args.backend.value)(module_name=module_name, include_dirs=args.include_dirs, include_paths=args.include_paths, external_resources=args.link_resource, debug=args.debug, arch_flags=args.arch, opt_flags=args.opt, f77_flags=args.f77flags, f90_flags=args.f90flags, linker_libpath=args.library_path, linker_libname=args.library_name, define_macros=args.define_macros, undef_macros=args.undef_macros) backend.compile(f77_files + f90_files, c_wrapper, build_dir) def main(): From 6f8ccbbeafa61c11ceeb4bca13d632e6d3054b41 Mon Sep 17 00:00:00 2001 From: Namami Shanker Date: Sat, 20 Aug 2022 11:53:32 +0530 Subject: [PATCH 114/132] ENH: Basic prototype meson backend --- numpy/f2py/backends/meson_backend.py | 57 ++++++++++++++++++++++--- numpy/f2py/backends/src/meson.build.src | 5 ++- 2 files changed, 54 insertions(+), 8 deletions(-) diff --git a/numpy/f2py/backends/meson_backend.py b/numpy/f2py/backends/meson_backend.py index b82e0bd2ee9d..5aa542621b9d 100644 --- a/numpy/f2py/backends/meson_backend.py +++ b/numpy/f2py/backends/meson_backend.py @@ -1,5 +1,6 @@ import os import errno +import shutil import subprocess from pathlib import Path @@ -8,7 +9,7 @@ class MesonTemplate: """Template meson build file generation class.""" - def __init__(self, module_name: str, numpy_install_path, numpy_get_include: Path, f2py_get_include: Path, c_wrapper: Path, fortran_sources: list[Path], dependencies: list[str], include_path: list[Path]): + def __init__(self, module_name: str, numpy_install_path, numpy_get_include: Path, f2py_get_include: Path, c_wrapper: Path, fortran_sources: list[Path], dependencies: list[str], include_path: list[Path], optimization_flags: list[str], architecture_flags: list[str], f77_flags: list[str], f90_flags: list[str], linker_libpath: list[Path], linker_libname: list[str], define_macros: list[tuple[str, str]], undef_macros: list[str]): self.module_name = module_name self.numpy_install_path = numpy_install_path self.build_template_path = numpy_install_path / "f2py" / "backends" / "src" / "meson.build.src" @@ -19,10 +20,20 @@ def __init__(self, module_name: str, numpy_install_path, numpy_get_include: Path self.dependencies = dependencies self.include_directories = include_path self.substitutions = {} + self.optimization_flags = optimization_flags + self.architecture_flags = architecture_flags + self.fortran_flags = f77_flags + f90_flags + self.linker_libpath = linker_libpath + self.linker_libname = linker_libname + self.define_macros = define_macros + self.undef_macros = undef_macros self.pipeline = [self.initialize_template, + self.global_flags_substitution, self.sources_substitution, self.dependencies_substitution, - self.include_directories_subtitution] + self.include_directories_subtitution, + self.linker_substitution, + self.macros_substitution] @property def meson_build_template(self) -> str: @@ -46,6 +57,28 @@ def dependencies_substitution(self) -> None: def include_directories_subtitution(self) -> None: self.substitutions["include_directories_list"] = ", ".join([f"include_directories('{include_directory}')" for include_directory in self.include_directories]) + def global_flags_substitution(self) -> None: + fortran_compiler_flags = self.fortran_flags + self.optimization_flags + self.architecture_flags + c_compiler_flags = self.optimization_flags + self.architecture_flags + self.substitutions["fortran_global_args"] = fortran_compiler_flags + self.substitutions["c_global_args"] = c_compiler_flags + + def macros_substitution(self) -> None: + self.substitutions["macros"] = "" + if self.define_macros: + self.substitutions["macros"] = ",".join(f"\'-D{macro[0]}={macro[1]}\'" if macro[1] else f"-D{macro[0]}" for macro in self.define_macros) + if self.undef_macros: + self.substitutions["macros"] += "," + ",".join(f"\'-U{macro}\'" for macro in self.undef_macros) + + def linker_substitution(self) -> None: + self.substitutions["linker_args"] = "" + if self.linker_libpath: + linker_libpath_subs = ",".join(f"-L{libpath}" for libpath in self.linker_libpath) + self.substitutions["linker_args"] += linker_libpath_subs + if self.linker_libname: + linker_libname_subs = ",".join(f"-l{libname}" for libname in self.linker_libname) + self.substitutions["linker_args"] += f",{linker_libname_subs}" + def generate_meson_build(self) -> str: for node in self.pipeline: node() @@ -56,6 +89,7 @@ def generate_meson_build(self) -> str: class MesonBackend(Backend): def __init__(self, module_name: str = 'untitled', fortran_compiler: str = None, c_compiler: str = None, f77exec: Path = None, f90exec: Path = None, f77_flags: list[str] = None, f90_flags: list[str] = None, include_paths: list[Path] = None, include_dirs: list[Path] = None, external_resources: list[str] = None, linker_libpath: list[Path] = None, linker_libname: list[str] = None, define_macros: list[tuple[str, str]] = None, undef_macros: list[str] = None, debug: bool = False, opt_flags: list[str] = None, arch_flags: list[str] = None, no_opt: bool = False, no_arch: bool = False) -> None: + self.meson_build_dir = "builddir" if f77_flags is None: f77_flags = [] if include_paths is None: @@ -96,11 +130,21 @@ def _get_optimization_level(self): def _set_environment_variables(self) -> None: if self.fortran_compiler: os.putenv("FC", self.fortran_compiler) + elif self.f77exec: + os.putenv("FC", self.f77exec) + elif self.f90exec: + os.putenv("FC", self.f90exec) if self.c_compiler: os.putenv("CC", self.c_compiler) + def _move_exec_to_root(self, build_dir: Path): + walk_dir = build_dir / self.meson_build_dir + path_objects = walk_dir.glob(f"{self.module_name}*.so") + for path_object in path_objects: + shutil.move(path_object, Path.cwd()) + def _get_build_command(self): - return ["meson", "setup", "builddir", "-Ddebug=true" if self.debug else "-Ddebug=false", f"-Doptimization={str(self._get_optimization_level())}"] + return ["meson", "setup", self.meson_build_dir, "-Ddebug=true" if self.debug else "-Ddebug=false", f"-Doptimization={str(self._get_optimization_level())}"] def load_wrapper(self, wrapper_path: Path) -> None: wrapper_path: Path = Path(wrapper_path) @@ -117,7 +161,7 @@ def load_sources(self, fortran_sources: list[Path]) -> None: def write_meson_build(self, build_dir: Path) -> None: """Writes the meson build file at specified location""" - meson_template = MesonTemplate(self.module_name, super().numpy_install_path(), self.numpy_get_include(), self.f2py_get_include(), self.c_wrapper, self.fortran_sources, self.external_resources, self.include_path+self.include_dirs) + meson_template = MesonTemplate(self.module_name, super().numpy_install_path(), self.numpy_get_include(), self.f2py_get_include(), self.c_wrapper, self.fortran_sources, self.external_resources, self.include_paths+self.include_dirs, optimization_flags=self.opt_flags, architecture_flags=self.arch_flags, f77_flags=self.f77_flags, f90_flags=self.f90_flags, linker_libpath=self.linker_libpath, linker_libname=self.linker_libname, define_macros=self.define_macros, undef_macros=self.undef_macros) src = meson_template.generate_meson_build() meson_build_file = build_dir / "meson.build" meson_build_file.write_text(src) @@ -128,14 +172,13 @@ def run_meson(self, build_dir: Path): completed_process = subprocess.run(self._get_build_command(), cwd=build_dir) if(completed_process.returncode != 0): raise subprocess.CalledProcessError(completed_process.returncode, completed_process.args) - completed_process = subprocess.run(["meson", "compile", "-C", "builddir"], cwd=build_dir) + completed_process = subprocess.run(["meson", "compile", "-C", self.meson_build_dir], cwd=build_dir) if(completed_process.returncode != 0): raise subprocess.CalledProcessError(completed_process.returncode, completed_process.args) - def compile(self, fortran_sources: list[Path], c_wrapper: Path, build_dir: Path) -> None: self.load_wrapper(c_wrapper) self.load_sources(fortran_sources) self.write_meson_build(build_dir) - self.write_meson_build(build_dir) self.run_meson(build_dir) + self._move_exec_to_root(build_dir) diff --git a/numpy/f2py/backends/src/meson.build.src b/numpy/f2py/backends/src/meson.build.src index 476e0d679303..c1991a5ef7f5 100644 --- a/numpy/f2py/backends/src/meson.build.src +++ b/numpy/f2py/backends/src/meson.build.src @@ -4,6 +4,9 @@ project('${modulename}', 'c', add_languages('fortran') +add_global_arguments(${c_global_args}, language: 'c') +add_global_arguments(${fortran_global_args}, language: 'fortran') + py_mod = import('python') py3 = py_mod.find_installation('python3') py3_dep = py3.dependency() @@ -16,4 +19,4 @@ incdir_f2py = '${f2py_get_include}' inc_np = include_directories(incdir_numpy, incdir_f2py) -py3.extension_module('${modulename}', '${c_wrapper}', ${source_list}, incdir_f2py+'/fortranobject.c', include_directories: [inc_np, ${include_directories_list}], dependencies : [py3_dep, ${dependencies_list}], install : true) \ No newline at end of file +py3.extension_module('${modulename}', '${c_wrapper}', ${source_list}, incdir_f2py+'/fortranobject.c', include_directories: [inc_np, ${include_directories_list}], dependencies : [py3_dep, ${dependencies_list}], link_args : '${linker_args}', c_args : [${macros}], install : true) \ No newline at end of file From 28414fae501fd44342fd87b21629f6999334eaca Mon Sep 17 00:00:00 2001 From: Namami Shanker Date: Wed, 24 Aug 2022 23:14:22 +0530 Subject: [PATCH 115/132] BUG: f2pyarg bugfixes --- numpy/f2py/f2pyarg.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/numpy/f2py/f2pyarg.py b/numpy/f2py/f2pyarg.py index 9fc0183bee0e..6a87291143fc 100644 --- a/numpy/f2py/f2pyarg.py +++ b/numpy/f2py/f2pyarg.py @@ -545,8 +545,7 @@ class Backends(enum.Enum): ) build_helpers.add_argument( - """_summary_ - """ "--noopt", + "--noopt", action="store_true", help="Compile without optimization" ) @@ -652,7 +651,7 @@ class Backends(enum.Enum): parser.add_argument('--backend', type=Backends, - default="distutils", + default=Backends.Distutils.value, action=EnumAction) # The rest, only works for files, since we expect: @@ -864,7 +863,7 @@ def process_args(args: argparse.Namespace, rem: list[str]) -> None: # Step 8: Generate wrapper or signature file if compile flag is not given c_wrapper = generate_files(f77_files + f90_files, module_name, sign_file) if c_wrapper and args.c: - backend: Backend = backends.get(args.backend.value)(module_name=module_name, include_dirs=args.include_dirs, include_paths=args.include_paths, external_resources=args.link_resource, debug=args.debug, arch_flags=args.arch, opt_flags=args.opt, f77_flags=args.f77flags, f90_flags=args.f90flags, linker_libpath=args.library_path, linker_libname=args.library_name, define_macros=args.define_macros, undef_macros=args.undef_macros) + backend: Backend = backends.get(args.backend)(module_name=module_name, include_dirs=args.include_dirs, include_paths=args.include_paths, external_resources=args.link_resource, debug=args.debug, arch_flags=args.arch, opt_flags=args.opt, f77_flags=args.f77flags, f90_flags=args.f90flags, linker_libpath=args.library_path, linker_libname=args.library_name, define_macros=args.define_macros, undef_macros=args.undef_macros) backend.compile(f77_files + f90_files, c_wrapper, build_dir) def main(): From 66fd50a2e87a51147eefc7a8eaa90c4a96175a38 Mon Sep 17 00:00:00 2001 From: Namami Shanker Date: Wed, 24 Aug 2022 23:14:43 +0530 Subject: [PATCH 116/132] MAINT: Return Path object of tempdir --- numpy/f2py/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/numpy/f2py/utils.py b/numpy/f2py/utils.py index 1fe95a81dde9..95c7ef12ad9c 100644 --- a/numpy/f2py/utils.py +++ b/numpy/f2py/utils.py @@ -28,7 +28,7 @@ def open_build_dir(build_dir: Optional[list[str]], compile: bool) -> Path: if build_dir is None: if compile: remove_build_dir = True - build_dir = tempfile.mkdtemp() + build_dir = Path(tempfile.mkdtemp()) else: build_dir = Path.cwd() else: From dd08d47f1c60b51989e793a1a9eaf6390adc8d3b Mon Sep 17 00:00:00 2001 From: Namami Shanker Date: Wed, 24 Aug 2022 23:16:12 +0530 Subject: [PATCH 117/132] ENH: Add F2PY backend directory to install config --- numpy/f2py/setup.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/numpy/f2py/setup.py b/numpy/f2py/setup.py index 499609f96600..cc4c19c08fb6 100644 --- a/numpy/f2py/setup.py +++ b/numpy/f2py/setup.py @@ -26,6 +26,9 @@ def configuration(parent_package='', top_path=None): config = Configuration('f2py', parent_package, top_path) config.add_subpackage('tests') + config.add_subpackage('backends') + config.add_data_dir('backends/src') + config.add_data_files('backends/src/meson.build.src') config.add_data_dir('tests/src') config.add_data_files( 'src/fortranobject.c', From 9ab0c9d4d4b231b2723709f3e3a4f0ccd20fad04 Mon Sep 17 00:00:00 2001 From: Namami Shanker Date: Thu, 25 Aug 2022 22:35:25 +0530 Subject: [PATCH 118/132] BUG: Generate using pyf files --- numpy/f2py/f2pyarg.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/numpy/f2py/f2pyarg.py b/numpy/f2py/f2pyarg.py index 6a87291143fc..ad141ec227bf 100644 --- a/numpy/f2py/f2pyarg.py +++ b/numpy/f2py/f2pyarg.py @@ -861,7 +861,7 @@ def process_args(args: argparse.Namespace, rem: list[str]) -> None: compile_dist(ext_args, link_resource, build_dir, fc_flags, flib_flags, args.quiet) else: # Step 8: Generate wrapper or signature file if compile flag is not given - c_wrapper = generate_files(f77_files + f90_files, module_name, sign_file) + c_wrapper = generate_files(f77_files + f90_files + pyf_files, module_name, sign_file) if c_wrapper and args.c: backend: Backend = backends.get(args.backend)(module_name=module_name, include_dirs=args.include_dirs, include_paths=args.include_paths, external_resources=args.link_resource, debug=args.debug, arch_flags=args.arch, opt_flags=args.opt, f77_flags=args.f77flags, f90_flags=args.f90flags, linker_libpath=args.library_path, linker_libname=args.library_name, define_macros=args.define_macros, undef_macros=args.undef_macros) backend.compile(f77_files + f90_files, c_wrapper, build_dir) From 706443c50b180afe498f669b2f4d9833216004ad Mon Sep 17 00:00:00 2001 From: Namami Shanker Date: Thu, 25 Aug 2022 22:36:30 +0530 Subject: [PATCH 119/132] ENH: Accept f77, f90 and object files seperately --- numpy/f2py/backends/meson_backend.py | 4 ++-- numpy/f2py/f2pyarg.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/numpy/f2py/backends/meson_backend.py b/numpy/f2py/backends/meson_backend.py index 5aa542621b9d..2a0c8a4791e9 100644 --- a/numpy/f2py/backends/meson_backend.py +++ b/numpy/f2py/backends/meson_backend.py @@ -176,9 +176,9 @@ def run_meson(self, build_dir: Path): if(completed_process.returncode != 0): raise subprocess.CalledProcessError(completed_process.returncode, completed_process.args) - def compile(self, fortran_sources: list[Path], c_wrapper: Path, build_dir: Path) -> None: + def compile(self, f77_sources: list[Path], f90_sources:list[Path], object_files: list[Path], c_wrapper: Path, build_dir: Path) -> None: self.load_wrapper(c_wrapper) - self.load_sources(fortran_sources) + self.load_sources(f77_sources + f90_sources + object_files) self.write_meson_build(build_dir) self.run_meson(build_dir) self._move_exec_to_root(build_dir) diff --git a/numpy/f2py/f2pyarg.py b/numpy/f2py/f2pyarg.py index ad141ec227bf..39672b40e93a 100644 --- a/numpy/f2py/f2pyarg.py +++ b/numpy/f2py/f2pyarg.py @@ -864,7 +864,7 @@ def process_args(args: argparse.Namespace, rem: list[str]) -> None: c_wrapper = generate_files(f77_files + f90_files + pyf_files, module_name, sign_file) if c_wrapper and args.c: backend: Backend = backends.get(args.backend)(module_name=module_name, include_dirs=args.include_dirs, include_paths=args.include_paths, external_resources=args.link_resource, debug=args.debug, arch_flags=args.arch, opt_flags=args.opt, f77_flags=args.f77flags, f90_flags=args.f90flags, linker_libpath=args.library_path, linker_libname=args.library_name, define_macros=args.define_macros, undef_macros=args.undef_macros) - backend.compile(f77_files + f90_files, c_wrapper, build_dir) + backend.compile(f77_files, f90_files, obj_files, c_wrapper, build_dir) def main(): logger = logging.getLogger("f2py_cli") From 4a114ce693bbf04663f57cc0cee2cc01a9287547 Mon Sep 17 00:00:00 2001 From: Namami Shanker Date: Thu, 25 Aug 2022 22:36:59 +0530 Subject: [PATCH 120/132] ENH: Add f2py backends to public API list --- numpy/tests/test_public_api.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/numpy/tests/test_public_api.py b/numpy/tests/test_public_api.py index 21ccea5a3aa9..63ce4e865007 100644 --- a/numpy/tests/test_public_api.py +++ b/numpy/tests/test_public_api.py @@ -246,6 +246,9 @@ def test_NPY_NO_EXPORT(): "distutils.unixccompiler", "dual", "f2py.auxfuncs", + "f2py.backends", + "f2py.backends.backend", + "f2py.backends.meson_backend", "f2py.capi_maps", "f2py.cb_rules", "f2py.cfuncs", From 26cc090c99b51641108ead1ace31774eda5bc1e0 Mon Sep 17 00:00:00 2001 From: Namami Shanker Date: Fri, 26 Aug 2022 15:32:57 +0530 Subject: [PATCH 121/132] BUG: Lower C wrapper defs by default for less buggy imports --- numpy/f2py/f2pyarg.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/numpy/f2py/f2pyarg.py b/numpy/f2py/f2pyarg.py index 39672b40e93a..1adb0a6f8861 100644 --- a/numpy/f2py/f2pyarg.py +++ b/numpy/f2py/f2pyarg.py @@ -318,7 +318,7 @@ class Backends(enum.Enum): "--no-lower", metavar="--[no-]lower", action=BoolAction, - default=False, + default=True, type=bool, help="""Do [not] lower the cases in . By default, --lower is assumed with -h key, and --no-lower without -h From ee41f1e974965241ae79e5dcc40141b5307bfb48 Mon Sep 17 00:00:00 2001 From: Namami Shanker Date: Fri, 26 Aug 2022 15:34:07 +0530 Subject: [PATCH 122/132] BUG: Read module name from pyf file without -c --- numpy/f2py/f2pyarg.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/numpy/f2py/f2pyarg.py b/numpy/f2py/f2pyarg.py index 1adb0a6f8861..90266d88faf8 100644 --- a/numpy/f2py/f2pyarg.py +++ b/numpy/f2py/f2pyarg.py @@ -731,12 +731,11 @@ def get_fortran_compiler_flags(args: argparse.Namespace) -> list[str]: def get_module_name(args: argparse.Namespace, pyf_files: list[str]) -> str: if(args.module is not None): return args.module[0] - if args.c: - for file in pyf_files: - if name := get_f2py_modulename(file): - return name - return "unititled" - return None + for file in pyf_files: + if name := get_f2py_modulename(file): + return name + if not name: + return "untitled" if args.c else None def get_signature_file(args: argparse.Namespace, build_dir: pathlib.Path) -> pathlib.Path: sign_file = None From 76ab74b7a026719dfb2186674b1bd23a15a501aa Mon Sep 17 00:00:00 2001 From: Namami Shanker Date: Fri, 26 Aug 2022 15:34:45 +0530 Subject: [PATCH 123/132] BUG: Fix enum action for switching backends --- numpy/f2py/f2pyarg.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/numpy/f2py/f2pyarg.py b/numpy/f2py/f2pyarg.py index 90266d88faf8..47c9ac27283a 100644 --- a/numpy/f2py/f2pyarg.py +++ b/numpy/f2py/f2pyarg.py @@ -651,7 +651,7 @@ class Backends(enum.Enum): parser.add_argument('--backend', type=Backends, - default=Backends.Distutils.value, + default=Backends.Distutils, action=EnumAction) # The rest, only works for files, since we expect: @@ -835,7 +835,7 @@ def process_args(args: argparse.Namespace, rem: list[str]) -> None: # Step 7: If user has asked for compilation. Mimic 'run_compile' from f2py2e # Disutils receives all the options and builds the extension. - if(args.c and args.backend == Backends.Distutils.value): + if(args.c and args.backend == Backends.Distutils): link_resource = args.link_resource # The 3 functions below generate arrays of flag similar to how @@ -862,7 +862,7 @@ def process_args(args: argparse.Namespace, rem: list[str]) -> None: # Step 8: Generate wrapper or signature file if compile flag is not given c_wrapper = generate_files(f77_files + f90_files + pyf_files, module_name, sign_file) if c_wrapper and args.c: - backend: Backend = backends.get(args.backend)(module_name=module_name, include_dirs=args.include_dirs, include_paths=args.include_paths, external_resources=args.link_resource, debug=args.debug, arch_flags=args.arch, opt_flags=args.opt, f77_flags=args.f77flags, f90_flags=args.f90flags, linker_libpath=args.library_path, linker_libname=args.library_name, define_macros=args.define_macros, undef_macros=args.undef_macros) + backend: Backend = backends.get(args.backend.value)(module_name=module_name, include_dirs=args.include_dirs, include_paths=args.include_paths, external_resources=args.link_resource, debug=args.debug, arch_flags=args.arch, opt_flags=args.opt, f77_flags=args.f77flags, f90_flags=args.f90flags, linker_libpath=args.library_path, linker_libname=args.library_name, define_macros=args.define_macros, undef_macros=args.undef_macros) backend.compile(f77_files, f90_files, obj_files, c_wrapper, build_dir) def main(): From bfd5c86d3b6187f8f86a493c4c170aa07193230d Mon Sep 17 00:00:00 2001 From: Namami Shanker Date: Mon, 29 Aug 2022 20:33:06 +0530 Subject: [PATCH 124/132] Revert "BUG: Read module name from pyf file without -c" This reverts commit 31ec0660526cf7b3374d5a4e89fe91508c7f16aa. --- numpy/f2py/f2pyarg.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/numpy/f2py/f2pyarg.py b/numpy/f2py/f2pyarg.py index 47c9ac27283a..520c6e8237ce 100644 --- a/numpy/f2py/f2pyarg.py +++ b/numpy/f2py/f2pyarg.py @@ -731,11 +731,12 @@ def get_fortran_compiler_flags(args: argparse.Namespace) -> list[str]: def get_module_name(args: argparse.Namespace, pyf_files: list[str]) -> str: if(args.module is not None): return args.module[0] - for file in pyf_files: - if name := get_f2py_modulename(file): - return name - if not name: - return "untitled" if args.c else None + if args.c: + for file in pyf_files: + if name := get_f2py_modulename(file): + return name + return "unititled" + return None def get_signature_file(args: argparse.Namespace, build_dir: pathlib.Path) -> pathlib.Path: sign_file = None From 24d7087b8a6acf24665ea1ebfd053b035f7e890c Mon Sep 17 00:00:00 2001 From: Namami Shanker Date: Wed, 31 Aug 2022 14:23:43 +0530 Subject: [PATCH 125/132] BUG: Change default value of lower flag --- numpy/f2py/f2pyarg.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/numpy/f2py/f2pyarg.py b/numpy/f2py/f2pyarg.py index 520c6e8237ce..d4a2c5303e3d 100644 --- a/numpy/f2py/f2pyarg.py +++ b/numpy/f2py/f2pyarg.py @@ -318,7 +318,7 @@ class Backends(enum.Enum): "--no-lower", metavar="--[no-]lower", action=BoolAction, - default=True, + default=False, type=bool, help="""Do [not] lower the cases in . By default, --lower is assumed with -h key, and --no-lower without -h From 46c466f98c860992c901afce6b034e26b879e19d Mon Sep 17 00:00:00 2001 From: Namami Shanker Date: Wed, 31 Aug 2022 14:25:18 +0530 Subject: [PATCH 126/132] BUG: Change default module name --- numpy/f2py/f2pyarg.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/numpy/f2py/f2pyarg.py b/numpy/f2py/f2pyarg.py index d4a2c5303e3d..626287e3dfde 100644 --- a/numpy/f2py/f2pyarg.py +++ b/numpy/f2py/f2pyarg.py @@ -736,7 +736,7 @@ def get_module_name(args: argparse.Namespace, pyf_files: list[str]) -> str: if name := get_f2py_modulename(file): return name return "unititled" - return None + return "" def get_signature_file(args: argparse.Namespace, build_dir: pathlib.Path) -> pathlib.Path: sign_file = None From 29c31ee78d21c55c3fd313a42a324ca34acd8718 Mon Sep 17 00:00:00 2001 From: Namami Shanker Date: Wed, 31 Aug 2022 14:28:41 +0530 Subject: [PATCH 127/132] MAINT: Set Default empty gen False --- numpy/f2py/f2pyarg.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/numpy/f2py/f2pyarg.py b/numpy/f2py/f2pyarg.py index 626287e3dfde..f257076f8c63 100644 --- a/numpy/f2py/f2pyarg.py +++ b/numpy/f2py/f2pyarg.py @@ -808,7 +808,7 @@ def process_args(args: argparse.Namespace, rem: list[str]) -> None: 'include_paths': args.include_paths, 'coutput': None, 'f2py_wrapper_output': None, - 'emptygen': True, + 'emptygen': False, } crackfortran_setts = { 'module': module_name, From 64fdd3e549f209cd92b3b6fb0743014fca41866d Mon Sep 17 00:00:00 2001 From: Namami Shanker Date: Wed, 31 Aug 2022 14:30:39 +0530 Subject: [PATCH 128/132] BUG: Return list of wrappers from generate_files --- numpy/f2py/backends/meson_backend.py | 21 +++++++++------------ numpy/f2py/backends/src/meson.build.src | 2 +- numpy/f2py/f2pyarg.py | 6 +++--- numpy/f2py/service.py | 15 +++++++++------ 4 files changed, 22 insertions(+), 22 deletions(-) diff --git a/numpy/f2py/backends/meson_backend.py b/numpy/f2py/backends/meson_backend.py index 2a0c8a4791e9..32381c4de368 100644 --- a/numpy/f2py/backends/meson_backend.py +++ b/numpy/f2py/backends/meson_backend.py @@ -9,14 +9,14 @@ class MesonTemplate: """Template meson build file generation class.""" - def __init__(self, module_name: str, numpy_install_path, numpy_get_include: Path, f2py_get_include: Path, c_wrapper: Path, fortran_sources: list[Path], dependencies: list[str], include_path: list[Path], optimization_flags: list[str], architecture_flags: list[str], f77_flags: list[str], f90_flags: list[str], linker_libpath: list[Path], linker_libname: list[str], define_macros: list[tuple[str, str]], undef_macros: list[str]): + def __init__(self, module_name: str, numpy_install_path, numpy_get_include: Path, f2py_get_include: Path, wrappers: list[Path], fortran_sources: list[Path], dependencies: list[str], include_path: list[Path], optimization_flags: list[str], architecture_flags: list[str], f77_flags: list[str], f90_flags: list[str], linker_libpath: list[Path], linker_libname: list[str], define_macros: list[tuple[str, str]], undef_macros: list[str]): self.module_name = module_name self.numpy_install_path = numpy_install_path self.build_template_path = numpy_install_path / "f2py" / "backends" / "src" / "meson.build.src" self.sources = fortran_sources self.numpy_get_include = numpy_get_include self.f2py_get_include = f2py_get_include - self.c_wrapper = c_wrapper + self.wrappers = wrappers self.dependencies = dependencies self.include_directories = include_path self.substitutions = {} @@ -49,7 +49,7 @@ def initialize_template(self) -> None: def sources_substitution(self) -> None: self.substitutions["source_list"] = ",".join(["\'"+str(source.absolute())+"\'" for source in self.sources]) - self.substitutions["c_wrapper"] = str(self.c_wrapper.absolute()) if self.c_wrapper else "" + self.substitutions["wrappers"] = ",".join(["\'"+str(wrapper.absolute())+"\'" for wrapper in self.wrappers]) def dependencies_substitution(self) -> None: self.substitutions["dependencies_list"] = ", ".join([f"dependency('{dependecy}')" for dependecy in self.dependencies]) @@ -116,7 +116,7 @@ def __init__(self, module_name: str = 'untitled', fortran_compiler: str = None, arch_flags = [] super().__init__(module_name, fortran_compiler, c_compiler, f77exec, f90exec, f77_flags, f90_flags, include_paths, include_dirs, external_resources, linker_libpath, linker_libname, define_macros, undef_macros, debug, opt_flags, arch_flags, no_opt, no_arch) - self.c_wrapper: Path = None + self.wrappers: list[Path] = [] self.fortran_sources: list[Path] = [] self.template = Template(self.fortran_sources) @@ -146,11 +146,8 @@ def _move_exec_to_root(self, build_dir: Path): def _get_build_command(self): return ["meson", "setup", self.meson_build_dir, "-Ddebug=true" if self.debug else "-Ddebug=false", f"-Doptimization={str(self._get_optimization_level())}"] - def load_wrapper(self, wrapper_path: Path) -> None: - wrapper_path: Path = Path(wrapper_path) - if not wrapper_path.is_file(): - raise FileNotFoundError(errno.ENOENT, f"{wrapper_path.absolute()} does not exist.") - self.c_wrapper = wrapper_path + def load_wrapper(self, wrappers: list[Path]) -> None: + self.wrappers = wrappers def load_sources(self, fortran_sources: list[Path]) -> None: for fortran_source in fortran_sources: @@ -161,7 +158,7 @@ def load_sources(self, fortran_sources: list[Path]) -> None: def write_meson_build(self, build_dir: Path) -> None: """Writes the meson build file at specified location""" - meson_template = MesonTemplate(self.module_name, super().numpy_install_path(), self.numpy_get_include(), self.f2py_get_include(), self.c_wrapper, self.fortran_sources, self.external_resources, self.include_paths+self.include_dirs, optimization_flags=self.opt_flags, architecture_flags=self.arch_flags, f77_flags=self.f77_flags, f90_flags=self.f90_flags, linker_libpath=self.linker_libpath, linker_libname=self.linker_libname, define_macros=self.define_macros, undef_macros=self.undef_macros) + meson_template = MesonTemplate(self.module_name, super().numpy_install_path(), self.numpy_get_include(), self.f2py_get_include(), self.wrappers, self.fortran_sources, self.external_resources, self.include_paths+self.include_dirs, optimization_flags=self.opt_flags, architecture_flags=self.arch_flags, f77_flags=self.f77_flags, f90_flags=self.f90_flags, linker_libpath=self.linker_libpath, linker_libname=self.linker_libname, define_macros=self.define_macros, undef_macros=self.undef_macros) src = meson_template.generate_meson_build() meson_build_file = build_dir / "meson.build" meson_build_file.write_text(src) @@ -176,8 +173,8 @@ def run_meson(self, build_dir: Path): if(completed_process.returncode != 0): raise subprocess.CalledProcessError(completed_process.returncode, completed_process.args) - def compile(self, f77_sources: list[Path], f90_sources:list[Path], object_files: list[Path], c_wrapper: Path, build_dir: Path) -> None: - self.load_wrapper(c_wrapper) + def compile(self, f77_sources: list[Path], f90_sources:list[Path], object_files: list[Path], wrappers: list[Path], build_dir: Path) -> None: + self.load_wrapper(wrappers) self.load_sources(f77_sources + f90_sources + object_files) self.write_meson_build(build_dir) self.run_meson(build_dir) diff --git a/numpy/f2py/backends/src/meson.build.src b/numpy/f2py/backends/src/meson.build.src index c1991a5ef7f5..42d33f973bc5 100644 --- a/numpy/f2py/backends/src/meson.build.src +++ b/numpy/f2py/backends/src/meson.build.src @@ -19,4 +19,4 @@ incdir_f2py = '${f2py_get_include}' inc_np = include_directories(incdir_numpy, incdir_f2py) -py3.extension_module('${modulename}', '${c_wrapper}', ${source_list}, incdir_f2py+'/fortranobject.c', include_directories: [inc_np, ${include_directories_list}], dependencies : [py3_dep, ${dependencies_list}], link_args : '${linker_args}', c_args : [${macros}], install : true) \ No newline at end of file +py3.extension_module('${modulename}', ${wrappers}, ${source_list}, incdir_f2py+'/fortranobject.c', include_directories: [inc_np, ${include_directories_list}], dependencies : [py3_dep, ${dependencies_list}], link_args : '${linker_args}', c_args : [${macros}], install : true) \ No newline at end of file diff --git a/numpy/f2py/f2pyarg.py b/numpy/f2py/f2pyarg.py index f257076f8c63..7a9a458b6b51 100644 --- a/numpy/f2py/f2pyarg.py +++ b/numpy/f2py/f2pyarg.py @@ -861,10 +861,10 @@ def process_args(args: argparse.Namespace, rem: list[str]) -> None: compile_dist(ext_args, link_resource, build_dir, fc_flags, flib_flags, args.quiet) else: # Step 8: Generate wrapper or signature file if compile flag is not given - c_wrapper = generate_files(f77_files + f90_files + pyf_files, module_name, sign_file) - if c_wrapper and args.c: + wrappers = generate_files(f77_files + f90_files + pyf_files, module_name, sign_file) + if wrappers and args.c: backend: Backend = backends.get(args.backend.value)(module_name=module_name, include_dirs=args.include_dirs, include_paths=args.include_paths, external_resources=args.link_resource, debug=args.debug, arch_flags=args.arch, opt_flags=args.opt, f77_flags=args.f77flags, f90_flags=args.f90flags, linker_libpath=args.library_path, linker_libname=args.library_name, define_macros=args.define_macros, undef_macros=args.undef_macros) - backend.compile(f77_files, f90_files, obj_files, c_wrapper, build_dir) + backend.compile(f77_files, f90_files, obj_files, wrappers, build_dir) def main(): logger = logging.getLogger("f2py_cli") diff --git a/numpy/f2py/service.py b/numpy/f2py/service.py index 4d18342fbc27..f49027751e32 100644 --- a/numpy/f2py/service.py +++ b/numpy/f2py/service.py @@ -270,7 +270,8 @@ def wrapper_settings(rules_setts: dict[str, Any], crackfortran_setts: dict[str, # 2. https://github.com/numpy/numpy/blob/main/numpy/f2py/f2py2e.py#L471-L473 _set_auxfuncs(auxfuncs_setts) -def generate_files(files: list[str], module_name: str, sign_file: Path) -> None: +def generate_files(files: list[str], module_name: str, sign_file: Path) -> list[Path]: + """Generate signature file if wanted and return list of wrappers to be compiled""" # Step 8.1: Generate postlist from crackfortran postlist = _callcrackfortran(files, module_name) @@ -282,11 +283,13 @@ def generate_files(files: list[str], module_name: str, sign_file: Path) -> None: # https://github.com/numpy/numpy/blob/main/numpy/f2py/f2py2e.py#L343-L350 _generate_signature(postlist, sign_file) return - if(module_name): - # Step 8.4: Same as the buildmodules folder of f2py2e - ret = _buildmodules(postlist) - c_wrapper = ret.get(module_name).get('csrc')[0] - return Path(c_wrapper) + # Step 8.4: Same as the buildmodules folder of f2py2e + ret = _buildmodules(postlist) + module_name = list(ret.keys())[0] + wrappers = [] + wrappers.extend(ret.get(module_name).get('csrc', [])) + wrappers.extend(ret.get(module_name).get('fsrc', [])) + return [Path(wrapper) for wrapper in wrappers] def compile_dist(ext_args: dict[str, Any], link_resources: list[str], build_dir: Path, fc_flags: list[str], flib_flags: list[str], quiet_build: bool) -> None: # Step 7.2: The entire code below mimics 'f2py2e:run_compile()' From 5888fbf740c8a050aa9e4ba79c7225bca91e2426 Mon Sep 17 00:00:00 2001 From: Namami Shanker Date: Wed, 31 Aug 2022 14:31:04 +0530 Subject: [PATCH 129/132] BUG: Get module name from postlist --- numpy/f2py/service.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/numpy/f2py/service.py b/numpy/f2py/service.py index f49027751e32..5a38aa445ff0 100644 --- a/numpy/f2py/service.py +++ b/numpy/f2py/service.py @@ -234,8 +234,8 @@ def _check_postlist(postlist: list[dict[str, Any]], sign_file: Path) -> None: def _callcrackfortran(files: list[str], module_name: str) -> list[dict[str, Any]]: postlist = crackfortran.crackfortran([str(file) for file in files]) for mod in postlist: - mod["coutput"] = f"{module_name}module.c" - mod["f2py_wrapper_output"] = f"{module_name}-f2pywrappers.f" + mod["coutput"] = f"{mod['name']}module.c" + mod["f2py_wrapper_output"] = f"{mod['name']}-f2pywrappers.f" return postlist def _set_dependencies_dist(ext_args: dict[str, Any], link_resource: list[str]) -> None: From 5664a9885a88f61c02e158548b90ac128b774c6a Mon Sep 17 00:00:00 2001 From: Namami Shanker Date: Wed, 7 Sep 2022 20:02:47 +0530 Subject: [PATCH 130/132] BUG: Shift files to the end of sys.argv --- numpy/f2py/f2pyarg.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/numpy/f2py/f2pyarg.py b/numpy/f2py/f2pyarg.py index 7a9a458b6b51..7ef55a6d8315 100644 --- a/numpy/f2py/f2pyarg.py +++ b/numpy/f2py/f2pyarg.py @@ -17,6 +17,7 @@ from __future__ import annotations +import sys import argparse import logging import os @@ -866,9 +867,16 @@ def process_args(args: argparse.Namespace, rem: list[str]) -> None: backend: Backend = backends.get(args.backend.value)(module_name=module_name, include_dirs=args.include_dirs, include_paths=args.include_paths, external_resources=args.link_resource, debug=args.debug, arch_flags=args.arch, opt_flags=args.opt, f77_flags=args.f77flags, f90_flags=args.f90flags, linker_libpath=args.library_path, linker_libname=args.library_name, define_macros=args.define_macros, undef_macros=args.undef_macros) backend.compile(f77_files, f90_files, obj_files, wrappers, build_dir) +def sort_args(args: list[str]) -> list[str]: + """Sort files at the end of the list""" + extensions = (".f", ".for", ".ftn", ".f77", ".f90", ".f95", ".f03", ".f08", ".pyf", ".src", ".o", ".out", ".so", ".a") + if any(arg.endswith(extensions) for arg in args): + return sorted(args, key=lambda arg: arg.endswith(extensions)) + def main(): logger = logging.getLogger("f2py_cli") logger.setLevel(logging.WARNING) + sys.argv = sort_args(sys.argv) args, rem = parser.parse_known_args() # since argparse can't handle '-include
' # we filter it out into rem and parse it manually. From a1b82193e9a405bf4c478c97abe4be0b708d0542 Mon Sep 17 00:00:00 2001 From: Namami Shanker Date: Thu, 8 Sep 2022 00:36:55 +0530 Subject: [PATCH 131/132] BUG: Import annotations for python3.8 --- numpy/f2py/backends/meson_backend.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/numpy/f2py/backends/meson_backend.py b/numpy/f2py/backends/meson_backend.py index 32381c4de368..f23798c97cb2 100644 --- a/numpy/f2py/backends/meson_backend.py +++ b/numpy/f2py/backends/meson_backend.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import os import errno import shutil From cf21444b65cd3007c53311ac43fe36eaf5c6eee4 Mon Sep 17 00:00:00 2001 From: Namami Shanker Date: Wed, 7 Sep 2022 21:14:53 +0530 Subject: [PATCH 132/132] BUG: f2pyarg sort args fix --- numpy/f2py/f2pyarg.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/numpy/f2py/f2pyarg.py b/numpy/f2py/f2pyarg.py index 7ef55a6d8315..847939187aa4 100644 --- a/numpy/f2py/f2pyarg.py +++ b/numpy/f2py/f2pyarg.py @@ -870,8 +870,7 @@ def process_args(args: argparse.Namespace, rem: list[str]) -> None: def sort_args(args: list[str]) -> list[str]: """Sort files at the end of the list""" extensions = (".f", ".for", ".ftn", ".f77", ".f90", ".f95", ".f03", ".f08", ".pyf", ".src", ".o", ".out", ".so", ".a") - if any(arg.endswith(extensions) for arg in args): - return sorted(args, key=lambda arg: arg.endswith(extensions)) + return sorted(args, key=lambda arg: arg.endswith(extensions)) def main(): logger = logging.getLogger("f2py_cli")