From ab962b592d8bd46698f93858a0f3d3a0d779ba17 Mon Sep 17 00:00:00 2001 From: Vincent Jacques Date: Wed, 12 Jun 2024 04:27:49 +0000 Subject: [PATCH] Import 'pybind11' instead running of a subprocess --- .../build-topics-test-gha-branch.yml | 384 +++++++++--------- setup.py | 13 +- 2 files changed, 197 insertions(+), 200 deletions(-) diff --git a/.github/workflows/build-topics-test-gha-branch.yml b/.github/workflows/build-topics-test-gha-branch.yml index 100a5e0d..30d60e7e 100644 --- a/.github/workflows/build-topics-test-gha-branch.yml +++ b/.github/workflows/build-topics-test-gha-branch.yml @@ -30,52 +30,52 @@ jobs: - make-source-dist strategy: matrix: - python_version: ['3.8', '3.12'] + python_version: ['3.8'] steps: - name: Install Python uses: actions/setup-python@v5 with: python-version: ${{ matrix.python_version }} - name: Install Python packages - run: pip${{ matrix.python_version }} install auditwheel build Chrones pybind11 twine - - name: Install CUDA - run: | - sudo apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64/3bf863cc.pub - sudo add-apt-repository "deb https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64/ /" - sudo apt-get update - sudo apt-get install --yes --no-install-recommends cuda-cudart-dev-12-1 cuda-nvcc-12-1 - echo "/usr/local/cuda-12.1/bin" >>$GITHUB_PATH - - name: Install Boost - run: | - cd /home/runner/work - wget https://boostorg.jfrog.io/artifactory/main/release/1.82.0/source/boost_1_82_0.tar.gz - tar xf boost_*.tar.gz - rm boost_*.tar.gz - cd boost_* - echo "using python : ${{ matrix.python_version }} ;" >tools/build/src/user-config.jam - ./bootstrap.sh - ./b2 --with-python python=${{ matrix.python_version }} link=shared variant=release stage - sudo cp -r boost /usr/local/include - sudo cp -r stage/lib/* /usr/local/lib - sudo ldconfig - - name: Install OR-Tools - run: | - cd /home/runner/work - wget https://github.com/google/or-tools/releases/download/v8.2/or-tools_ubuntu-20.04_v8.2.8710.tar.gz - tar xf or-tools_*.tar.gz - rm or-tools_*.tar.gz - cd or-tools_* - sudo cp -r include/* /usr/local/include - sudo cp -r lib/* /usr/local/lib - sudo ldconfig - - name: Install patchelf - run: | - cd /home/runner/work - mkdir patchelf - cd patchelf - wget https://github.com/NixOS/patchelf/releases/download/0.18.0/patchelf-0.18.0-x86_64.tar.gz - tar xf *.tar.gz - sudo cp bin/patchelf /usr/local/bin + run: pip${{ matrix.python_version }} install auditwheel build Chrones pybind11 twine wheel + # - name: Install CUDA + # run: | + # sudo apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64/3bf863cc.pub + # sudo add-apt-repository "deb https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64/ /" + # sudo apt-get update + # sudo apt-get install --yes --no-install-recommends cuda-cudart-dev-12-1 cuda-nvcc-12-1 + # echo "/usr/local/cuda-12.1/bin" >>$GITHUB_PATH + # - name: Install Boost + # run: | + # cd /home/runner/work + # wget https://boostorg.jfrog.io/artifactory/main/release/1.82.0/source/boost_1_82_0.tar.gz + # tar xf boost_*.tar.gz + # rm boost_*.tar.gz + # cd boost_* + # echo "using python : ${{ matrix.python_version }} ;" >tools/build/src/user-config.jam + # ./bootstrap.sh + # ./b2 --with-python python=${{ matrix.python_version }} link=shared variant=release stage + # sudo cp -r boost /usr/local/include + # sudo cp -r stage/lib/* /usr/local/lib + # sudo ldconfig + # - name: Install OR-Tools + # run: | + # cd /home/runner/work + # wget https://github.com/google/or-tools/releases/download/v8.2/or-tools_ubuntu-20.04_v8.2.8710.tar.gz + # tar xf or-tools_*.tar.gz + # rm or-tools_*.tar.gz + # cd or-tools_* + # sudo cp -r include/* /usr/local/include + # sudo cp -r lib/* /usr/local/lib + # sudo ldconfig + # - name: Install patchelf + # run: | + # cd /home/runner/work + # mkdir patchelf + # cd patchelf + # wget https://github.com/NixOS/patchelf/releases/download/0.18.0/patchelf-0.18.0-x86_64.tar.gz + # tar xf *.tar.gz + # sudo cp bin/patchelf /usr/local/bin - name: Dowload the source distribution from GitHub Actions artifacts uses: actions/download-artifact@v4 @@ -87,10 +87,10 @@ jobs: rm *.tar.gz - name: Build the wheel - run: python${{ matrix.python_version }} -m build --wheel --outdir local-dist lincs-* - env: - LINCS_DEV_FORCE_NVCC: "true" - LINCS_DEV_FORCE_CHRONES: "true" + run: python${{ matrix.python_version }} -m build --no-isolation --wheel --outdir local-dist lincs-* + # env: + # LINCS_DEV_FORCE_NVCC: "true" + # LINCS_DEV_FORCE_CHRONES: "true" - name: Make the wheel machine-independent run: auditwheel repair --plat manylinux_2_31_x86_64 --strip local-dist/*.whl --wheel-dir dist - name: Check the wheel @@ -102,158 +102,158 @@ jobs: name: wheel-dist-${{ matrix.python_version }}-linux path: dist - build-for-windows: - runs-on: windows-2019 - needs: - - build-for-linux - strategy: - matrix: - python_version: ['3.8', '3.12'] - steps: - - name: Install Python - uses: actions/setup-python@v5 - with: - python-version: ${{ matrix.python_version }} - - name: Install Python packages - run: pip install build delvewheel pybind11 twine - - name: Install CUDA - uses: Jimver/cuda-toolkit@v0.2.14 - id: cuda-toolkit - with: - cuda: '12.1.0' - use-github-cache: false - use-local-cache: false - method: network - # Package names from https://docs.nvidia.com/cuda/cuda-installation-guide-microsoft-windows/index.html#id2 - sub-packages: '["nvcc", "cudart"]' - - name: Install MSys2 - uses: msys2/setup-msys2@v2 - with: - update: true - msystem: UCRT64 - install: >- - gzip - tar - unzip - wget - - name: Install boost - shell: msys2 {0} - run: | - cd /d/a - wget https://boostorg.jfrog.io/artifactory/main/release/1.82.0/source/boost_1_82_0.tar.gz - tar xf boost_*.tar.gz - rm boost_*.tar.gz - mv boost_* boost - cd boost - echo "using python : ${{ matrix.python_version }} ;" >tools/build/src/user-config.jam - ./bootstrap.bat - ./b2 --with-python python=${{ matrix.python_version }} link=shared variant=release stage || true - mkdir -p /d/lincs-deps/include /d/lincs-deps/lib - cp -r boost /d/lincs-deps/include - cp -r stage/lib/* /d/lincs-deps/lib - - name: Install OR-Tools - shell: msys2 {0} - run: | - cd /d/a - wget https://github.com/google/or-tools/releases/download/v8.2/or-tools_VisualStudio2019-64bit_v8.2.8710.zip - unzip or-tools_*.zip - rm or-tools_*.zip - cd or-tools_* - cp -r include/* /d/lincs-deps/include - cp -r lib/* /d/lincs-deps/lib + # build-for-windows: + # runs-on: windows-2019 + # needs: + # - build-for-linux + # strategy: + # matrix: + # python_version: ['3.8', '3.12'] + # steps: + # - name: Install Python + # uses: actions/setup-python@v5 + # with: + # python-version: ${{ matrix.python_version }} + # - name: Install Python packages + # run: pip install build delvewheel pybind11 twine + # - name: Install CUDA + # uses: Jimver/cuda-toolkit@v0.2.14 + # id: cuda-toolkit + # with: + # cuda: '12.1.0' + # use-github-cache: false + # use-local-cache: false + # method: network + # # Package names from https://docs.nvidia.com/cuda/cuda-installation-guide-microsoft-windows/index.html#id2 + # sub-packages: '["nvcc", "cudart"]' + # - name: Install MSys2 + # uses: msys2/setup-msys2@v2 + # with: + # update: true + # msystem: UCRT64 + # install: >- + # gzip + # tar + # unzip + # wget + # - name: Install boost + # shell: msys2 {0} + # run: | + # cd /d/a + # wget https://boostorg.jfrog.io/artifactory/main/release/1.82.0/source/boost_1_82_0.tar.gz + # tar xf boost_*.tar.gz + # rm boost_*.tar.gz + # mv boost_* boost + # cd boost + # echo "using python : ${{ matrix.python_version }} ;" >tools/build/src/user-config.jam + # ./bootstrap.bat + # ./b2 --with-python python=${{ matrix.python_version }} link=shared variant=release stage || true + # mkdir -p /d/lincs-deps/include /d/lincs-deps/lib + # cp -r boost /d/lincs-deps/include + # cp -r stage/lib/* /d/lincs-deps/lib + # - name: Install OR-Tools + # shell: msys2 {0} + # run: | + # cd /d/a + # wget https://github.com/google/or-tools/releases/download/v8.2/or-tools_VisualStudio2019-64bit_v8.2.8710.zip + # unzip or-tools_*.zip + # rm or-tools_*.zip + # cd or-tools_* + # cp -r include/* /d/lincs-deps/include + # cp -r lib/* /d/lincs-deps/lib - - name: Dowload the source distribution from GitHub Actions artifacts - uses: actions/download-artifact@v4 - with: - name: source-dist - - name: Unzip the source distribution - shell: msys2 {0} - run: | - tar xf *.tar.gz - rm *.tar.gz + # - name: Dowload the source distribution from GitHub Actions artifacts + # uses: actions/download-artifact@v4 + # with: + # name: source-dist + # - name: Unzip the source distribution + # shell: msys2 {0} + # run: | + # tar xf *.tar.gz + # rm *.tar.gz - - name: Build the wheel - run: python -m build --wheel --outdir local-dist (get-item lincs-*) - env: - LINCS_DEV_FORCE_NVCC: "true" - LINCS_DEV_DEPENDENCIES: d:\lincs-deps - LINCS_DEV_VC_VERSION: 142 - - name: Make the wheel machine-independent - run: delvewheel repair --strip (get-item local-dist\*.whl) --wheel-dir dist --add-path d:\lincs-deps\lib - - name: Check the wheel - run: twine check (get-item dist\*.whl) + # - name: Build the wheel + # run: python -m build --wheel --outdir local-dist (get-item lincs-*) + # env: + # LINCS_DEV_FORCE_NVCC: "true" + # LINCS_DEV_DEPENDENCIES: d:\lincs-deps + # LINCS_DEV_VC_VERSION: 142 + # - name: Make the wheel machine-independent + # run: delvewheel repair --strip (get-item local-dist\*.whl) --wheel-dir dist --add-path d:\lincs-deps\lib + # - name: Check the wheel + # run: twine check (get-item dist\*.whl) - - name: Upload the wheel to GitHub Actions artifacts - uses: actions/upload-artifact@v4 - with: - name: wheel-dist-${{ matrix.python_version }}-windows - path: dist + # - name: Upload the wheel to GitHub Actions artifacts + # uses: actions/upload-artifact@v4 + # with: + # name: wheel-dist-${{ matrix.python_version }}-windows + # path: dist - build-for-macos: - runs-on: macos-11 - needs: - - build-for-windows - strategy: - matrix: - python_version: ['3.8', '3.12'] - env: - MACOSX_DEPLOYMENT_TARGET: 11.0 - steps: - - name: Install Python - uses: actions/setup-python@v5 - with: - python-version: ${{ matrix.python_version }} - - name: Install Python packages - run: pip3 install build delocate pybind11 twine - - name: Install OpenMP - run: | - cd /Users/runner/work - mkdir openmp - cd openmp - wget https://mac.r-project.org/openmp/openmp-10.0.0-darwin17-Release.tar.gz - tar xf *.tar.gz - sudo cp usr/local/lib/* /usr/local/lib - sudo cp usr/local/include/* /usr/local/include - - name: Install Boost - run: | - cd /Users/runner/work - wget https://boostorg.jfrog.io/artifactory/main/release/1.82.0/source/boost_1_82_0.tar.gz - tar xf boost_*.tar.gz - rm boost_*.tar.gz - cd boost_* - echo "using python : ${{ matrix.python_version }} ;" >tools/build/src/user-config.jam - ./bootstrap.sh - ./b2 --with-python python=${{ matrix.python_version }} link=shared variant=release stage - sudo cp -r boost /usr/local/include - sudo cp -r stage/lib/* /usr/local/lib - - name: Install OR-Tools - run: | - cd /Users/runner/work - wget https://github.com/google/or-tools/releases/download/v8.2/or-tools_MacOsX-11.2.1_v8.2.8710.tar.gz - tar xf or-tools_*.tar.gz - rm or-tools_*.tar.gz - cd or-tools_* - sudo cp -r include/* /usr/local/include - sudo cp -r lib/* /usr/local/lib + # build-for-macos: + # runs-on: macos-11 + # needs: + # - build-for-windows + # strategy: + # matrix: + # python_version: ['3.8', '3.12'] + # env: + # MACOSX_DEPLOYMENT_TARGET: 11.0 + # steps: + # - name: Install Python + # uses: actions/setup-python@v5 + # with: + # python-version: ${{ matrix.python_version }} + # - name: Install Python packages + # run: pip3 install build delocate pybind11 twine + # - name: Install OpenMP + # run: | + # cd /Users/runner/work + # mkdir openmp + # cd openmp + # wget https://mac.r-project.org/openmp/openmp-10.0.0-darwin17-Release.tar.gz + # tar xf *.tar.gz + # sudo cp usr/local/lib/* /usr/local/lib + # sudo cp usr/local/include/* /usr/local/include + # - name: Install Boost + # run: | + # cd /Users/runner/work + # wget https://boostorg.jfrog.io/artifactory/main/release/1.82.0/source/boost_1_82_0.tar.gz + # tar xf boost_*.tar.gz + # rm boost_*.tar.gz + # cd boost_* + # echo "using python : ${{ matrix.python_version }} ;" >tools/build/src/user-config.jam + # ./bootstrap.sh + # ./b2 --with-python python=${{ matrix.python_version }} link=shared variant=release stage + # sudo cp -r boost /usr/local/include + # sudo cp -r stage/lib/* /usr/local/lib + # - name: Install OR-Tools + # run: | + # cd /Users/runner/work + # wget https://github.com/google/or-tools/releases/download/v8.2/or-tools_MacOsX-11.2.1_v8.2.8710.tar.gz + # tar xf or-tools_*.tar.gz + # rm or-tools_*.tar.gz + # cd or-tools_* + # sudo cp -r include/* /usr/local/include + # sudo cp -r lib/* /usr/local/lib - - name: Dowload the source distribution from GitHub Actions artifacts - uses: actions/download-artifact@v4 - with: - name: source-dist - - name: Unzip the source distribution - run: | - tar xf *.tar.gz - rm *.tar.gz + # - name: Dowload the source distribution from GitHub Actions artifacts + # uses: actions/download-artifact@v4 + # with: + # name: source-dist + # - name: Unzip the source distribution + # run: | + # tar xf *.tar.gz + # rm *.tar.gz - - name: Build the wheel - run: python3 -m build --wheel --outdir local-dist lincs-* - - name: Make the wheel machine-independent - run: delocate-wheel --wheel-dir dist local-dist/*.whl - - name: Check the wheel - run: twine check dist/*.whl + # - name: Build the wheel + # run: python3 -m build --wheel --outdir local-dist lincs-* + # - name: Make the wheel machine-independent + # run: delocate-wheel --wheel-dir dist local-dist/*.whl + # - name: Check the wheel + # run: twine check dist/*.whl - - name: Upload the wheel to GitHub Actions artifacts - uses: actions/upload-artifact@v4 - with: - name: wheel-dist-${{ matrix.python_version }}-macos - path: dist + # - name: Upload the wheel to GitHub Actions artifacts + # uses: actions/upload-artifact@v4 + # with: + # name: wheel-dist-${{ matrix.python_version }}-macos + # path: dist diff --git a/setup.py b/setup.py index 3f2da521..920e8b3c 100644 --- a/setup.py +++ b/setup.py @@ -225,14 +225,11 @@ def make_liblincs_extension(): print("WARNING: 'chrones' was not found, lincs will be compiled without Chrones", file=sys.stderr) try: - include_dirs += [ - option[2:] - for option in subprocess.run( - [sys.executable, "-m", "pybind11", "--includes"], capture_output=True, universal_newlines=True, check=True, - ).stdout.strip().split(" ") - ] - except subprocess.CalledProcessError: - print("WARNING: 'pybind11' was not found, compilation will fail", file=sys.stderr) + import pybind11 + except ImportError: + print("WARNING: 'pybind11' was not found, compilation will fail. (Source distribution is OK)", file=sys.stderr) + else: + include_dirs += [pybind11.get_include()] if sys.platform == "linux": extra_compile_args["c++"] = ["-std=c++17", "-Werror=switch", "-fopenmp"]