Switch to pyproject.toml based build system
jameskermode opened this issue · 7 comments
Distutils and numpy.distutils are deprecated. We should switch to a declarative build system, e.g. Meson as scipy has done.
Would you be interested in CMake as a potential path?
I got f2py working with pure CMake (bypassing distutils, as suggested on the numpy.f2py page). Same thing applies to f90wrap; I ran f2py-f90wrap with only "-m" (no "-c") which allows injecting the 3 modifications to numpy.f2py rules into the C code; building the .so happens through the linker instead of compiling with distutils
I got the f2py-only solution working with all 3 operating systems (linux, mac, windows)
Static libraries are also generated because I can pass those specific flags to the linker - something that was not possible with f2py's distutils wrapper (missing --extra-link-args) - so builds are decently portable though still specific to python 3.X and numpy 1.Y. On target machines, the dependencies on libgcc (or equivalent) are removed. Confirmed that it works for builds created in virtual environments too. Finally, for backwards compatibility, I added some utilities to substitute for the get_include() functions in older versions of numpy.
Testing my f90wrap+cmake build (less distutils calls) for windows. When I have it cleaned up I'll post it as a utility script in a PR if you think its useful
Thanks, would be good to see the example. Happy to support as a possible path, but I don't think it should be the default or only route.
If you want to follow numpy/scipy to Meson it should be fairly simple to do.
Copy pasting a cmake function below. It is a general-purpose wrapper to call f90wrap's scripts with a user-specified (CMake) list of f90 source files, static libraries and "include" locations (including fortran modules) to build a python (3) wrapper for fortran source. A generalization + adaptation to f90wrap for the baseline f2py CMake wrapper given here.
#==============================================================
# Function to run f90wrap on a list of source files
#==============================================================
function (f90wrap_nodistutils compiler F90FLAGS src_list prj libs locs)
# This cmake function runs f90wrap without distutils to wrap fortran code
# Arguments are
# 1. f90wrap_no_distutils: name of the function (in cmake, this is the first "argument")
# 2. compiler : name of fortran compiler ('gfortran', 'pgfortran', ..)
# 3. F90FLAGS : fortran source compilation flags
# 4. src_list : list of fortran sources to be wrapped with f90wrap (.f90, .F90 files)
# 5. prj : .so / .pyd module file name (script prepends an underscore)
# 6. libs : list of compiled library files (.a)
# 7. locs : list of paths to include while building the shared object
# Grab Python version
IF(NOT DEFINED PYVERSION)
message(" ")
message(" --- FATAL ERROR: INCORRECT SYNTAX DETECTED FOR CMAKE INVOCATION")
MESSAGE("Suggested remedy 1: invoke using 'cmake -DPYVERSION=3.X.Y ../../'")
MESSAGE("Suggested remedy 2: invoke using 'cmake -DPYVERSION=3.X ../../'")
message(FATAL_ERROR "PYVERSION is not set: CMAKE WILL EXIT")
message(" ")
ELSE()
find_package(Python 3.${PYVERSION} EXACT REQUIRED
COMPONENTS Interpreter Development.Module NumPy)
ENDIF()
include_directories(${PYTHON_INCLUDE_DIR})
# get F2PY_INCLUDE_DIR by querying location of numpy with pathlib
# dont use get_include() from numpy because it doesnt exist in numpy versions <1.21.1
# running it this way ensures that if running in virtual environments, the right version
# of numpy is selected
execute_process(
COMMAND "${Python_EXECUTABLE}"
-c "import pathlib, numpy.f2py as f2py; print(pathlib.Path(f2py.__file__).resolve().parent / 'src')"
OUTPUT_VARIABLE F2PY_INCLUDE_DIR
OUTPUT_STRIP_TRAILING_WHITESPACE
)
# this command does nothing on *nix. On windows, replaces "\" with "/" in f2py include directory
string(REPLACE "\\" "/" F2PY_INCLUDE_DIR ${F2PY_INCLUDE_DIR})
# get F90WRAP_INCLUDE_DIR by querying location of numpy with pathlib
# doesnt seem to be used anywhere so leave it commented out
# dont use get_include() from numpy because it doesnt exist in numpy versions <1.21.1
# execute_process(
# COMMAND "${Python_EXECUTABLE}"
# -c "import pathlib, f90wrap; print(pathlib.Path(f90wrap.__file__).resolve().parent)"
# OUTPUT_VARIABLE F90WRAP_INCLUDE_DIR
# OUTPUT_STRIP_TRAILING_WHITESPACE
# )
# # for windows: replace \ with / in path
# string(REPLACE "\\" "/" F90WRAP_INCLUDE_DIR ${F90WRAP_INCLUDE_DIR})
# Print out the discovered paths
include(CMakePrintHelpers)
cmake_print_variables(Python_INCLUDE_DIRS)
cmake_print_variables(F2PY_INCLUDE_DIR)
# cmake_print_variables(F90WRAP_INCLUDE_DIR)
cmake_print_variables(Python_NumPy_INCLUDE_DIRS)
# First step: make directory to place f90wrap-generated files
SET(path ${BIN}/${prj})
IF(EXISTS ${path})
ELSE()
FILE(MAKE_DIRECTORY ${path})
ENDIF(EXISTS ${path})
set(wrapdir ${path}/f90wrap)
IF(EXISTS ${wrapdir})
ELSE()
FILE(MAKE_DIRECTORY ${wrapdir})
ENDIF(EXISTS ${wrapdir})
# Include file with preprocessor cmake function definitions
INCLUDE(${CMAKE_MODULE_PATH}/Preprocess_definition.cmake)
# Identify compiler and set flags (function defined in Preprocess_definition.cmake)
id_flags("${compiler}" "${flags}" "${PAR_FLAG}" "${OMP}" "${CN}" "${EXT}")
# Build list of preprocessed file names
set(preproc_files "")
set(PREPROC_targets "")
set(f90wrap_src ${wrapdir}/f90wrap_toplevel.f90)
# Add custom target (dummy tag PREPROC) that tells CMake ${file_list} is the
# list of ependencies needed to make it (Without custom target, preprocessor will not run)
add_custom_target(
PREPROC ALL
COMMENT "Preprocessing files"
VERBATIM)
# loop over files that need to be wrapped with f90wrap
FOREACH(filename ${src_list})
# each file will be processed by f90wrap -m and a corresponding file called
# f90wrap_{$filename} will be created; build list of those files
# get filename without path or extensions
get_filename_component(name ${filename} NAME_WE)
# add to list of files matching pattern f90wrap_*.f90
list(APPEND f90wrap_src ${wrapdir}/f90wrap_${name}.f90)
# Define preprocessed file name with full path (in BIN dir)
set(fpp_file ${wrapdir}/${name}${EXT})
# add .fpp file to list of such files
LIST(APPEND preproc_files ${fpp_file}) # preprocessed files
# Tell CMake to run the preprocessor with a custom command
add_custom_command(
OUTPUT ${fpp_file}
COMMAND ${FC} ${flags} ${filename} -o ${fpp_file}
COMMENT "PREPROCESSING FORTRAN FILE ${filename}"
VERBATIM)
# add a target that ensures it is indeed generated
add_custom_target(PREPROC_${name} ALL
DEPENDS ${fpp_file})
# COMMENT "PREPROCESSING FILE ${filename}")
# remember this target to a list of targets
LIST(APPEND PREPROC_targets PREPROC_${name}) # preprocessed files
# tell CMake that creating PREPROC_${prj} requires preprocessor outputs
add_dependencies(PREPROC PREPROC_${name})
# tag this source file (.fpp) as being a generated output
set_source_files_properties(${fpp_file} PROPERTIES GENERATED TRUE)
ENDFOREACH(filename)
MESSAGE("PREPROC FILE_LIST IS " ${preproc_files})
MESSAGE("f90wrap_src IS " ${f90wrap_src})
# Step 2: run f90wrap on source files to create API defs in .py
#MESSAGE("OUTPUT OF STAGE 1 F90wrap is " ${output})
set(kmap_file "${CMAKE_MODULE_PATH}/kind_map")
# Add a custom command for running f90wrap
add_custom_command(
OUTPUT OUTPUT1
COMMAND ${Python_EXECUTABLE} -m f90wrap.scripts.main -m ${prj} -k ${kmap_file} ${preproc_files} -v
COMMENT "CREATING PYTHON API FILE"
WORKING_DIRECTORY ${wrapdir}
DEPENDS PREPROC
)
# add custom target to run the command above
add_custom_target(
API_${prj} ALL
DEPENDS OUTPUT1)
# COMMENT "creating python API"
# VERBATIM)
# touch all files in ${f90wrap_src} (files to be compiled by gfortran)
# If they exist, timestamp is slightly modified (harmless)
# If they dont exist, blank files are created so we can pass the static list
# without having to guess what files are eliminated by f90wrap -m call
add_custom_target(TOUCHD_f90wrap_files ALL)
FOREACH(filename ${f90wrap_src})
get_filename_component(name ${filename} NAME_WE)
add_custom_command(
OUTPUT TOUCH_${name}
COMMAND ${CMAKE_COMMAND} -E touch ${filename}
)
add_custom_target(
TOUCHD_${name}
DEPENDS TOUCH_${name}
)
add_dependencies(TOUCHD_f90wrap_files TOUCHD_${name})
ENDFOREACH()
add_dependencies(TOUCHD_f90wrap_files API_${prj})
# run f2py-f90wrap in "-m" mode only to invoke f2py -m command with f90wrap modifications
set(SO _${prj})
# setup f2py build directory
set(f2py_build ${wrapdir}/f2py_build)
IF(EXISTS ${f2py_build})
ELSE()
FILE(MAKE_DIRECTORY ${f2py_build})
ENDIF(EXISTS ${f2py_build})
# Common variables
set(f2py_module_name ${SO})
set(f2py_module_c "${f2py_build}/${f2py_module_name}module.c")
# Generate sources from f90wrap
add_custom_target(
genpyf ALL
DEPENDS ${f2py_module_c}
)
add_custom_command(
OUTPUT ${f2py_module_c}
# new C code generator: f2py_f90wrap (has interrupt handler)
COMMAND ${Python_EXECUTABLE} -m f90wrap.scripts.f2py_f90wrap -m ${SO} ${f90wrap_src} --lower
# old C code generator: f2py (works fine too)
# COMMAND ${Python_EXECUTABLE} -m numpy.f2py -m ${SO} ${f90wrap_src} --lower # Important
DEPENDS ${f90wrap_src} # recompile based on changes to these fortran source files
WORKING_DIRECTORY ${f2py_build} # create files in this folder
)
# move the created .py file -> bin directory (copy + remove)
add_custom_command(TARGET API_${prj}
POST_BUILD
COMMAND ${CMAKE_COMMAND} -E copy ${prj}.py ${BIN}
COMMAND ${CMAKE_COMMAND} -E remove ${prj}.py
WORKING_DIRECTORY ${wrapdir})
# Set up target module
set_source_files_properties(${f90wrap_src} PROPERTIES GENERATED TRUE)
set_source_files_properties(${f2py_module_c} PROPERTIES GENERATED TRUE)
# Note: libraries are created in ${LIB} variable
Python_add_library(${SO} MODULE WITH_SOABI
${f2py_module_c} # Generated
${F2PY_INCLUDE_DIR}/fortranobject.c # From NumPy
${f90wrap_src} # Fortran source(s) - generated by f90wrap -m
)
# Depend on sources
target_link_libraries(${SO} PRIVATE Python::NumPy)
target_include_directories(${SO} PRIVATE "${F2PY_INCLUDE_DIR}" "${PYTHON_INCLUDE_DIR}")
# add static libraries
foreach(lib_name ${libs})
target_link_libraries(${SO} PRIVATE ${lib_name})
endforeach()
# add locations for static libraries
foreach(lib_loc ${locs})
target_include_directories(${SO} PRIVATE ${lib_loc})
endforeach()
# compile SO only after <>module.c file is generated by f90wrap/f2py
add_dependencies(${SO} genpyf)
# add target compilation options
target_compile_options(${SO} PRIVATE ${F90FLAGS})
set_target_properties(${SO} PROPERTIES LINK_FLAGS "-static -static-libgcc -static-libgfortran")
add_custom_target(
SHAREDOBJECT_${prj} ALL
DEPENDS ${SO}
)
endfunction()
If you want to follow numpy/scipy to Meson it should be fairly simple to do.
Thanks - yes, this seems like the best course of action. We recently converted another package, matscipy to meson and it was relatively painless, but that didn't have any Fortran code, just C++.
If you have already used Meson in a mixed Python/Fortran project and would be willing to attempt the conversion then a PR would be very welcome - or just a pointer to a simpler example Meson build file than the full numpy/scipy setup.
EDIT: I suppose this is a good starting point https://numpy.org/doc/stable/f2py/buildtools/meson.html
Copy pasting a cmake function below. It is a general-purpose wrapper to call f90wrap's scripts with a user-specified (CMake) list of f90 source files, static libraries and "include" locations (including fortran modules) to build a python (3) wrapper for fortran source. A generalization + adaptation to f90wrap for the baseline f2py CMake wrapper given here.
Thanks for sharing this. I'd be happy to include it as an example, if you would be willing to make a PR. We should exclude it from the default test-set to avoid a mandatory dependency on cmake, but would be good to include it in the GitHub CI tests to ensure it doesn't break in future.
Sure. I actually did this last night, so I can PR it as soon as I get home today.