External Library: Link PFFT to FFTW when using MKL

Issue

MKL does not contain a full set of compiled API for FFTW.

Suggestions from Sebastian

There is MKL's custom shared object builder. We use this to extract the BLAS, LAPACK, and ScaLAPACK functionality out of MKL.

To link against this, you should be able to do something like:

export SCALAPACK_PKGCONFIG="$(mktemp -d)"
cat <<EOF > "${SCALAPACK_PKGCONFIG}/scalapack.pc"
Name: scalapack
Description: Intel(R) Math Kernel Library
Version: 1
Libs: -L${MKL_PARTS_HOME}/lib -lmkl_cluster -lm -ldl
EOF

and pass

      -DBLAS_LIBRARIES="${MKL_PARTS_HOME}/lib/libmkl_blas.so" \
      -DLAPACK_LIBRARIES="${MKL_PARTS_HOME}/lib/libmkl_lapack.so" \

to the cmake call.

With this approach, we build octopus+MKL+FFTW+pfft+libvdwxc-mpi.

Prior Attempted Solution

One can manually build the FFTW wrappers, create a static lib from them, and link these to any target that depends on them.
Below is what I attempted to do along these lines, however a single function is missing from MKL's FFTW wrappers that pfft required. I assumed this was a naming inconsistency and tried writing a trivial wrapper. In doing so, Octopus compiles but the pfft tests segfault when creating an FFTW plan:

Using intel2023a-mpi

module purge
module load toolchain/intel2023a-mpi
module load octopus-dependencies/full
module rm pnfft/1.0.7-alpha
unset CPATH 
unset LIBRARY_PATH

cmake --preset ci-intel-min-mpi --fresh -DCMAKE_REQUIRE_FIND_PACKAGE_pfft=On -DCMAKE_DISABLE_FIND_PACKAGE_pfft=Off -Wno-deprecated
cmake --build cmake-build-ci-intel-min-mpi -j 
cmake --test-dir cmake-build-ci-intel-min-mpi -R pfft

where this preset is introduced in MR !2789 (merged).

CMakeLists.txt

option(OCTOPUS_MKL_FFTW "Octopus: Build with MKL's FFTW-MPI wrappers" OFF)
.
.
.

if (OCTOPUS_MKL)
	find_package(MKL CONFIG)
	set_package_properties(MKL PROPERTIES TYPE REQUIRED)

	if (OCTOPUS_MKL_FFTW)
		# Build the MKL FFTW-MPI wrappers as a lib 
		# See https://www.intel.com/content/www/us/en/docs/onemkl/developer-reference-fortran/2024-1/mpi-fftw3-wrappers.html
		# and https://www.intel.com/content/www/us/en/docs/onemkl/developer-reference-fortran/2024-1/building-your-own-wrapper-library.html
		set(MKL_FFTW_MPI_WRAPPERS_DIR ${MKL_ROOT}/interfaces/fftw3x_cdft/wrappers)
		file(GLOB MKL_FFTW_MPI_WRAPPERS ${MKL_FFTW_MPI_WRAPPERS_DIR}/*.c)
		# Debugging: Print out detected wrapper files
		message(STATUS "MKL FFTW-MPI Wrappers: ${MKL_FFTW_MPI_WRAPPERS}")

		# This function does not exist, so assume it is a naming issue and wrap the MKL function
		# https://community.intel.com/t5/Intel-oneAPI-Math-Kernel-Library/Missing-symbol-in-FFTW-MPI-interface-fftw-mpi-execute-r2r/m-p/1179043
		add_library(MKL_FFTW_MPI_WRAPPERS STATIC ${MKL_FFTW_MPI_WRAPPERS} ${CMAKE_CURRENT_SOURCE_DIR}/cmake/fftw_mpi_execute_r2r.c)
		target_include_directories(MKL_FFTW_MPI_WRAPPERS PRIVATE ${MKL_ROOT}/include ${MKL_ROOT}/include/fftw ${MKL_ROOT}/interfaces/fftw3x_cdft/wrappers)
		target_link_libraries(MKL_FFTW_MPI_WRAPPERS PRIVATE MPI::MPI_C)
		target_compile_definitions(MKL_FFTW_MPI_WRAPPERS PRIVATE MKL_DOUBLE)

		# Ensure symbols are not optimized out
		target_link_options(MKL_FFTW_MPI_WRAPPERS PRIVATE -Wl,--whole-archive)
	endif ()

elseif (OCTOPUS_FFTW)

cmake/Findpfft.cmake

# pfft requires FFTW with MPI support but this dependency is not handled by pfft's build system.
# See https://github.com/mpip/pfft/issues/40. As such, the consuming project must explicitly provide FFTW::DoubleMPI
# or MKL's interface to FFTW.
if (pfft_FOUND)
    if (NOT TARGET MKL::MKL AND NOT TARGET FFTW::DoubleMPI)
        message(FATAL_ERROR "pfft was found, but FFTW with MPI support, which is required by pfft, is not available.")
    endif ()
endif ()

cmake/fftw_mpi_execute_r2r.c

#include "fftw3-mpi_mkl.h"
#include "mpi_transpose.h"

// Avoid  warning #266: function "fftw_mpi_execute_dft_r2r" declared implicitly
void fftw_mpi_execute_dft_r2r(fftw_plan p, double *in, double *out);

void fftw_mpi_execute_r2r(fftw_plan p, double *in, double *out) {
    fftw_mpi_execute_dft_r2r(p, in, out);
}

src/math/CMakeLists.txt

if (TARGET MKL::MKL)
	target_link_libraries(Octopus_lib PRIVATE MKL::MKL)

	if (OCTOPUS_MKL_FFTW)
		# TODO If this works, create an option to build FFTW-MPI-MKL, and move pfft_with_mkl to below
		add_library(pfft_with_mkl INTERFACE)
		target_link_libraries(pfft_with_mkl INTERFACE 
			pfft::pfft 
			MKL_FFTW_MPI_WRAPPERS
			${MKL_ROOT}/lib/intel64/libmkl_cdft_core.so
		)
		target_include_directories(pfft_with_mkl INTERFACE ${MKL_ROOT}/include/fftw ${MKL_ROOT}/interfaces/fftw3x_cdft/wrappers)

		target_link_libraries(Octopus_lib PRIVATE 
			pfft_with_mkl
			MKL::MKL
		)
	else ()
		target_link_libraries(Octopus_lib PRIVATE MKL::MKL)
	endif ()
.
.
.
if (TARGET pfft::pfft)
	target_link_libraries(Octopus_lib PRIVATE pfft::pfft)
	if (NOT OCTOPUS_MKL) 
		# pfft requires FFTW::DoubleMPI but the external project's build system does not
		# find/link to it. See https://github.com/mpip/pfft/issues/40. As such, we do the linking.
		# Imported targets cannot be modified, so link FFTW::DoubleMPI directly to Octopus_lib
		target_link_libraries(Octopus_lib PRIVATE pfft::pfft FFTW::DoubleMPI)
	endif()
endif ()