From a6092253a16a6d4ca818d3a876e04faced851659 Mon Sep 17 00:00:00 2001
From: Ian Harry <ian.harry@ligo.org>
Date: Mon, 12 Jun 2023 13:27:02 +0100
Subject: [PATCH] Try to fix CI tests (#4394)

* Switch FFTW/MKL order of preference

* Swap backend order and force gomp import

* FIx typo

* Do need openmpi in tox.ini

* Fix bug in MKL error function

* Force FFTW on this one.

* CC

* Requested Josh changes
---
 examples/inference/lisa_smbhb_inj/run.sh | 5 +++++
 pycbc/fft/backend_cpu.py                 | 2 +-
 pycbc/fft/fftw.py                        | 9 +++++++--
 pycbc/fft/mkl.py                         | 2 +-
 tox.ini                                  | 1 +
 5 files changed, 15 insertions(+), 4 deletions(-)

diff --git a/examples/inference/lisa_smbhb_inj/run.sh b/examples/inference/lisa_smbhb_inj/run.sh
index 6c1f3edfd76..9ae22f7774c 100644
--- a/examples/inference/lisa_smbhb_inj/run.sh
+++ b/examples/inference/lisa_smbhb_inj/run.sh
@@ -3,4 +3,9 @@ pycbc_inference \
 --config-files `dirname "$0"`/lisa_smbhb_relbin.ini \
 --output-file lisa_smbhb_inj_pe.hdf \
 --force \
+--fft-backends fftw \
 --verbose
+
+# PLEASE NOTE: This example is currently forcing a FFTW backend because MKL
+#              seems to fail for FFT lengths > 2^24. This is fine for most LIGO
+#              applications, but an issue for most LISA applications.
diff --git a/pycbc/fft/backend_cpu.py b/pycbc/fft/backend_cpu.py
index 6901ef0ebef..3a5c96a1ea8 100644
--- a/pycbc/fft/backend_cpu.py
+++ b/pycbc/fft/backend_cpu.py
@@ -20,7 +20,7 @@
 _backend_dict = {'fftw' : 'fftw',
                  'mkl' : 'mkl',
                  'numpy' : 'npfft'}
-_backend_list = ['fftw','mkl','numpy']
+_backend_list = ['mkl', 'fftw', 'numpy']
 
 _alist, _adict = _list_available(_backend_list, _backend_dict)
 
diff --git a/pycbc/fft/fftw.py b/pycbc/fft/fftw.py
index 3c164d998b3..4c9a5c8ed4d 100644
--- a/pycbc/fft/fftw.py
+++ b/pycbc/fft/fftw.py
@@ -106,6 +106,10 @@ def _init_threads(backend):
         raise ValueError("Backend {0} for FFTW threading does not exist!".format(backend))
     if double_threaded_libname is not None:
         try:
+            # For reasons Ian doesn't understand we should not load libgomp
+            # first using RTLD_DEEPBIND, so force loading it here if needed
+            if backend == 'openmp':
+                get_ctypes_library('gomp', [], mode=ctypes.DEFAULT_MODE)
             # Note that the threaded libraries don't have their own pkg-config
             # files we must look for them wherever we look for double or single
             # FFTW itself.
@@ -152,8 +156,9 @@ def set_threads_backend(backend=None):
         if retval != 0:
             raise RuntimeError("Could not initialize FFTW threading backend {0}".format(backend))
     else:
-        # Note that we pop() from the end, so 'openmp' is the first thing tried
-        _backend_list = ['unthreaded','pthreads','openmp']
+        # Note that we pop() from the end, so 'pthreads'
+        # is the first thing tried
+        _backend_list = ['unthreaded','openmp', 'pthreads']
         while not _fftw_threaded_set:
             _next_backend = _backend_list.pop()
             retval = _init_threads(_next_backend)
diff --git a/pycbc/fft/mkl.py b/pycbc/fft/mkl.py
index 987ae4bc65c..6bd75ee0c9c 100644
--- a/pycbc/fft/mkl.py
+++ b/pycbc/fft/mkl.py
@@ -69,8 +69,8 @@ def check_status(status):
     there is an error.
     """
     if status:
+        lib.DftiErrorMessage.restype = ctypes.c_char_p
         msg = lib.DftiErrorMessage(status)
-        msg = ctypes.c_char_p(msg).value
         raise RuntimeError(msg)
 
 def create_descriptor(size, idtype, odtype, inplace):
diff --git a/tox.ini b/tox.ini
index 67aa9f7fda2..e41f67686d2 100644
--- a/tox.ini
+++ b/tox.ini
@@ -62,6 +62,7 @@ conda_deps=
     binutils_linux-64>=2.39
     gsl
     lapack==3.6.1
+    openmpi
 conda_channels=conda-forge
 setenv = PYCBC_TEST_TYPE=docs
 commands = bash tools/pycbc_test_suite.sh