From 0a2850ade834b8c0072c296f1320795c38081cb7 Mon Sep 17 00:00:00 2001 From: Werner Van Geit Date: Wed, 27 Dec 2023 08:41:46 +0100 Subject: [PATCH 1/5] Working wheel for python environment --- .gitignore | 1 + CMakeLists.txt | 21 +++++++++++++++------ Makefile | 4 ++-- pyproject.toml | 16 ++++++++++------ scripts/fix_wheel.sh | 20 ++++++++++++++++++++ 5 files changed, 48 insertions(+), 14 deletions(-) create mode 100755 scripts/fix_wheel.sh diff --git a/.gitignore b/.gitignore index c7c4c39..4ed5823 100644 --- a/.gitignore +++ b/.gitignore @@ -247,3 +247,4 @@ pyrightconfig.json build.sh dakota dakota_wheel-* +wheelhouse diff --git a/CMakeLists.txt b/CMakeLists.txt index 720a646..adfc4bd 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,24 +1,33 @@ cmake_minimum_required(VERSION 3.15) project(itis_dakota) +add_definitions(-w) + +# set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wall -lpthread -lutil") + +find_package(Threads REQUIRED) + set(PY_BUILD_CMAKE_BIN_DIRECTORY ${PY_BUILD_CMAKE_PACKAGE_NAME}-${PY_BUILD_CMAKE_PACKAGE_VERSION}.data/scripts) set(PY_BUILD_CMAKE_LIB_DIRECTORY ${PY_BUILD_CMAKE_PACKAGE_NAME}-${PY_BUILD_CMAKE_PACKAGE_VERSION}.data/platlib) add_subdirectory(dakota EXCLUDE_FROM_ALL) -add_custom_target(dakota_for_python ALL DEPENDS environment dakota_src dakota) +add_custom_target(dakota_for_python ALL DEPENDS environment dakota_src ${Dakota_TPL_LIBRARIES}) - -install(TARGETS dakota - RUNTIME - COMPONENT dakota_for_python - DESTINATION ${PY_BUILD_CMAKE_BIN_DIRECTORY}) +#install(TARGETS dakota +# RUNTIME +# COMPONENT dakota_for_python +# DESTINATION ${PY_BUILD_CMAKE_BIN_DIRECTORY}) install(TARGETS dakota_src LIBRARY COMPONENT dakota_for_python LIBRARY DESTINATION ${PY_BUILD_CMAKE_BIN_DIRECTORY}) +#install(PROGRAMS ${Dakota_TPL_LIBRARIES} +# COMPONENT dakota_for_python +# DESTINATION ${PY_BUILD_CMAKE_LIB_DIRECTORY}/dakota/environment/) + install(TARGETS environment LIBRARY COMPONENT dakota_for_python diff --git a/Makefile b/Makefile index 9f26422..f1ad113 100644 --- a/Makefile +++ b/Makefile @@ -1,8 +1,8 @@ all: -wheel: - pip wheel -v . +wheel: clean + CIBW_BUILD=cp38*x86_64 CIBW_BUILD_VERBOSITY=10 CIBW_SKIP="*musllinux*" cibuildwheel --platform linux install: pip install -v . diff --git a/pyproject.toml b/pyproject.toml index fd10bb1..18cb1a1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,16 +9,20 @@ requires = ["py-build-cmake", "numpy"] build-backend = "py_build_cmake.build" [tool.cibuildwheel] -before-build = ["pushd /opt/_internal && tar -xJf static-libs-for-embedding-only.tar.xz && popd", "pip install numpy", "pip install cmake --upgrade"] -before-all = ["yum install -y gcc-gfortran lapack-devel boost169-devel boost169-filesystem boost169-program-options boost169-regex boost169-serialization blas-devel"] -repair-wheel-command = "auditwheel repair -w {dest_dir} {wheel}" +before-build = ["pip install auditwheel --upgrade", "pip install numpy", "pip install cmake --upgrade"] +#before-build = ["pip install auditwheel --upgrade", "pushd /opt/_internal && tar -xJf static-libs-for-embedding-only.tar.xz && popd", "pip install numpy", "pip install cmake --upgrade"] +#before-build = ["pip install numpy", "pip install cmake --upgrade"] +before-all = ["yum install -y lapack-static boost169-static blas-static python3-devel.x86_64"] -[tool.py-build-cmake.sdist] -include = ["dakota/*", "CMakeLists.txt", "LICENSE.txt"] +repair-wheel-command = "auditwheel repair -w {dest_dir} {wheel} && ./scripts/fix_wheel.sh {dest_dir}" + +# [tool.py-build-cmake.sdist] +# include = ["dakota/*", "CMakeLists.txt", "LICENSE.txt"] [tool.py-build-cmake.cmake] source_path = "." -options = {"DAKOTA_PYTHON_WRAPPER" = "ON", "BUILD_SHARED_LIBS"="OFF", "CMAKE_SKIP_INSTALL_RPATH" = "ON", "CMAKE_POSITION_INDEPENDENT_CODE" = "ON", "DAKOTA_PYTHON_DIRECT_INTERFACE" = "ON", "BOOST_LIBRARYDIR" = "/usr/lib64/boost169", "BOOST_INCLUDEDIR" = "/usr/include/boost169"} +#options = {"Boost_USE_STATIC_LIBS" = "ON", "DAKOTA_PYTHON_WRAPPER" = "ON", "CMAKE_POSITION_INDEPENDENT_CODE" = "ON", "BUILD_SHARED_LIBS"="OFF", "DAKOTA_PYTHON_DIRECT_INTERFACE" = "ON", "BOOST_LIBRARYDIR" = "/usr/lib64/boost169", "BOOST_INCLUDEDIR" = "/usr/include/boost169", "PYTHON_EXECUTABLE" = "/opt/python/cp38-cp38/bin/python", "PYTHON_INCLUDE_DIR" = "/opt/python/cp38-cp38/include", "PYTHON_LIBRARY" = "/opt/python/cp38-cp38/lib/libpython3.8.a", "PYTHON_LIBRARIES" = "/opt/python/cp38-cp38/lib/"} +options = {"DAKOTA_PYTHON_WRAPPER" = "ON", "CMAKE_POSITION_INDEPENDENT_CODE" = "ON", "BUILD_SHARED_LIBS"="OFF", "DAKOTA_PYTHON_DIRECT_INTERFACE" = "ON", "BOOST_LIBRARYDIR" = "/usr/lib64/boost169", "BOOST_INCLUDEDIR" = "/usr/include/boost169", "PYTHON_EXECUTABLE" = "/opt/python/cp38-cp38/bin/python"} build_args = ["-j"] build_type = "Release" install_components = ["dakota_for_python"] diff --git a/scripts/fix_wheel.sh b/scripts/fix_wheel.sh new file mode 100755 index 0000000..a99032a --- /dev/null +++ b/scripts/fix_wheel.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +set -ex + +WHEEL_DIR=$1 + +WHEEL_NAME=$(basename $(ls ${WHEEL_DIR}/*.whl) .whl) + +echo "Fixing ${WHEEL_DIR}/${WHEEL_NAME}.whl" + +cd ${WHEEL_DIR} + +unzip ${WHEEL_DIR}/${WHEEL_NAME}.whl -d ${WHEEL_NAME} + +find ${WHEEL_NAME} -type f -name "*.so" -exec patchelf --set-rpath '$ORIGIN/../../itis_dakota.libs' '{}' \; +find ${WHEEL_NAME}/itis_dakota.libs -type f -name "*.so.*" -exec patchelf --set-rpath '$ORIGIN/' '{}' \; + +cd ${WHEEL_NAME} +zip -r ${WHEEL_NAME}.whl * +mv ${WHEEL_NAME}.whl .. From 1818591e5412a0c57050220bff49bda9d658dbcc Mon Sep 17 00:00:00 2001 From: Werner Van Geit Date: Wed, 27 Dec 2023 08:50:48 +0100 Subject: [PATCH 2/5] Remove beforeall from github workflow --- .github/workflows/build-wheels.yml | 2 +- pyproject.toml | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/build-wheels.yml b/.github/workflows/build-wheels.yml index 07b6c30..1600131 100644 --- a/.github/workflows/build-wheels.yml +++ b/.github/workflows/build-wheels.yml @@ -28,7 +28,7 @@ jobs: - uses: pypa/cibuildwheel@v2.16 env: CIBW_BUILD: ${{ matrix.python }}*${{ matrix.arch }} - CIBW_BEFORE_ALL: "yum install -y gcc-gfortran lapack-devel boost169-devel boost169-filesystem boost169-program-options boost169-regex boost169-serialization blas-devel" + #CIBW_BEFORE_ALL: "yum install -y gcc-gfortran lapack-devel boost169-devel boost169-filesystem boost169-program-options boost169-regex boost169-serialization blas-devel" - name: Upload wheels uses: actions/upload-artifact@v3 with: diff --git a/pyproject.toml b/pyproject.toml index 18cb1a1..7b65438 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -11,7 +11,6 @@ build-backend = "py_build_cmake.build" [tool.cibuildwheel] before-build = ["pip install auditwheel --upgrade", "pip install numpy", "pip install cmake --upgrade"] #before-build = ["pip install auditwheel --upgrade", "pushd /opt/_internal && tar -xJf static-libs-for-embedding-only.tar.xz && popd", "pip install numpy", "pip install cmake --upgrade"] -#before-build = ["pip install numpy", "pip install cmake --upgrade"] before-all = ["yum install -y lapack-static boost169-static blas-static python3-devel.x86_64"] repair-wheel-command = "auditwheel repair -w {dest_dir} {wheel} && ./scripts/fix_wheel.sh {dest_dir}" From 2c9f373472635ee7b43552e207f742f742ae4c58 Mon Sep 17 00:00:00 2001 From: Werner Van Geit Date: Wed, 27 Dec 2023 09:29:14 +0100 Subject: [PATCH 3/5] Patch dakota source to remove embedded python --- Makefile | 3 ++- dakota-src.patch | 21 +++++++++++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) create mode 100644 dakota-src.patch diff --git a/Makefile b/Makefile index f1ad113..10ff7a3 100644 --- a/Makefile +++ b/Makefile @@ -19,4 +19,5 @@ get-dakota-src: cd dakota && \ git submodule update --init packages/external && \ git submodule update --init packages/pecos && \ - git submodule update --init packages/surfpack + git submodule update --init packages/surfpack && \ + git apply ../dakota-src.patch diff --git a/dakota-src.patch b/dakota-src.patch new file mode 100644 index 0000000..b1655b8 --- /dev/null +++ b/dakota-src.patch @@ -0,0 +1,21 @@ +diff --git a/cmake/DakotaFindPython.cmake b/cmake/DakotaFindPython.cmake +index 43f9fb9..f4c4e28 100644 +--- a/cmake/DakotaFindPython.cmake ++++ b/cmake/DakotaFindPython.cmake +@@ -8,7 +8,7 @@ macro(dakota_find_python) + if(DAKOTA_PYTHON_DIRECT_INTERFACE OR DAKOTA_PYTHON_SURROGATES OR + DAKOTA_PYTHON_WRAPPER OR DAKOTA_PYBIND11) + message(STATUS "Dakota enabling Python (Development) for direct or surrogate interface") +- list(APPEND dakota_python_components Development) ++ list(APPEND dakota_python_components Development.Module) + + if (DAKOTA_PYTHON_DIRECT_INTERFACE_NUMPY) + message(STATUS "Dakota enabling Python direct interface with NumPy") +@@ -18,6 +18,7 @@ macro(dakota_find_python) + endif() + + find_package(Python REQUIRED ${dakota_python_components}) ++ # find_package(Python COMPONENTS ${dakota_python_components} REQUIRED) + + # pybind11, C3, Acro, etc., use older CMake FindPythonInterp, so we + # coerce it to use same as Dakota; more complex situations may From efa8b65dbba9cb9be3ee9e578fb4269caa9340ea Mon Sep 17 00:00:00 2001 From: Werner Van Geit Date: Wed, 27 Dec 2023 09:52:11 +0100 Subject: [PATCH 4/5] Limit build jobs --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 7b65438..20094bf 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -22,6 +22,6 @@ repair-wheel-command = "auditwheel repair -w {dest_dir} {wheel} && ./scripts/fix source_path = "." #options = {"Boost_USE_STATIC_LIBS" = "ON", "DAKOTA_PYTHON_WRAPPER" = "ON", "CMAKE_POSITION_INDEPENDENT_CODE" = "ON", "BUILD_SHARED_LIBS"="OFF", "DAKOTA_PYTHON_DIRECT_INTERFACE" = "ON", "BOOST_LIBRARYDIR" = "/usr/lib64/boost169", "BOOST_INCLUDEDIR" = "/usr/include/boost169", "PYTHON_EXECUTABLE" = "/opt/python/cp38-cp38/bin/python", "PYTHON_INCLUDE_DIR" = "/opt/python/cp38-cp38/include", "PYTHON_LIBRARY" = "/opt/python/cp38-cp38/lib/libpython3.8.a", "PYTHON_LIBRARIES" = "/opt/python/cp38-cp38/lib/"} options = {"DAKOTA_PYTHON_WRAPPER" = "ON", "CMAKE_POSITION_INDEPENDENT_CODE" = "ON", "BUILD_SHARED_LIBS"="OFF", "DAKOTA_PYTHON_DIRECT_INTERFACE" = "ON", "BOOST_LIBRARYDIR" = "/usr/lib64/boost169", "BOOST_INCLUDEDIR" = "/usr/include/boost169", "PYTHON_EXECUTABLE" = "/opt/python/cp38-cp38/bin/python"} -build_args = ["-j"] +build_args = ["-j4"] build_type = "Release" install_components = ["dakota_for_python"] From 9f548f88511d1281254b679f79a1dd108db3cc48 Mon Sep 17 00:00:00 2001 From: Werner Van Geit Date: Wed, 27 Dec 2023 10:15:01 +0100 Subject: [PATCH 5/5] Disable musllinux --- .github/workflows/build-wheels.yml | 1 + Makefile | 2 +- pyproject.toml | 2 ++ 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build-wheels.yml b/.github/workflows/build-wheels.yml index 1600131..4b01fad 100644 --- a/.github/workflows/build-wheels.yml +++ b/.github/workflows/build-wheels.yml @@ -32,4 +32,5 @@ jobs: - name: Upload wheels uses: actions/upload-artifact@v3 with: + name: wheels-${{ matrix.os }}-${{ matrix.python }}-${{ matrix.arch }} path: wheelhouse/*.whl diff --git a/Makefile b/Makefile index 10ff7a3..52df720 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ all: -wheel: clean +wheel: cache-clean clean CIBW_BUILD=cp38*x86_64 CIBW_BUILD_VERBOSITY=10 CIBW_SKIP="*musllinux*" cibuildwheel --platform linux install: diff --git a/pyproject.toml b/pyproject.toml index 20094bf..e40ee6d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -14,6 +14,8 @@ before-build = ["pip install auditwheel --upgrade", "pip install numpy", "pip in before-all = ["yum install -y lapack-static boost169-static blas-static python3-devel.x86_64"] repair-wheel-command = "auditwheel repair -w {dest_dir} {wheel} && ./scripts/fix_wheel.sh {dest_dir}" +build-verbosity = 2 +skip = "*-musllinux_*" # [tool.py-build-cmake.sdist] # include = ["dakota/*", "CMakeLists.txt", "LICENSE.txt"]