Refactor build system

.github/workflows/cd.yml

@@ -24,118 +24,71 @@ on:
 
 jobs:
 
-  release-builds:
-    name: Build wheel for ${{ matrix.platform }} ${{ matrix.arch }}
-    timeout-minutes: 10
+  build-wheels:
+    name: Build all wheels
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.12'
+      - name: Install dev dependencies
+        run: |
+            python -m pip install --upgrade pip build hatchling requests twine
+      - name: Build wheels (and sdist)
+        run: python tools/build_all_wheels.py
+      - name: Twine check
+        run: |
+            twine check dist/*
+      - name: Upload distributions
+        uses: actions/upload-artifact@v4
+        with:
+          path: dist
+          name: dist
+
+  test-wheels:
+    name: Test wheel for ${{ matrix.name }}
+    needs: [build-wheels]
     runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false
       matrix:
         include:
-          - platform: windows
-            arch: AMD64
-            os: windows-latest
-            testable: true
-          - platform: windows
-            arch: ARM64
-            os: windows-latest
-          - platform: windows
-            arch: x86
+          - name: windows amd64
             os: windows-latest
-          - platform: macos
-            arch: arm64
+          - name: macos arm64
             os: macos-latest
-            testable: true
-          - platform: macos
-            arch: x86_64
+          - name: macos x86_64
             os: macos-13  # last Intel MacOS
-            cibw_version: '==2.16'  # delocation does not work for later versions
-          - platform: linux
-            arch: x86_64
-            os: ubuntu-latest
-            testable: true
-          - platform: linux
-            arch: aarch64
+          - name: linux amd64
             os: ubuntu-latest
     steps:
     - uses: actions/checkout@v4
-    - name: Set up QEMU
-      if: matrix.platform == 'linux' && matrix.arch == 'aarch64'
-      uses: docker/setup-qemu-action@v3
-      with:
-        platforms: arm64
-    - name: Install dev dependencies
-      run: |
-          python -m pip install --upgrade pip wheel setuptools twine cibuildwheel${{ matrix.cibw_version}}
-    - name: Build wheels
-      run: python -m cibuildwheel --output-dir dist
-      env:
-        CIBW_PLATFORM: ${{ matrix.platform }}
-        CIBW_ARCHS: ${{ matrix.arch }}
-    - name: Twine check
-      run: |
-          twine check dist/*
-    - name: Test wheel
-      if: matrix.testable
-      shell: bash
-      run: |
-          rm -rf ./wgpu
-          filename=$(ls dist/*.whl)
-          pip install $filename
-          pushd $HOME
-          python -c 'import wgpu.backends.wgpu_native; print(wgpu.backends.wgpu_native._ffi.lib_path)'
-          popd
-          pip uninstall -y wgpu
-          git reset --hard HEAD
-    - name: Upload distributions
-      uses: actions/upload-artifact@v4
-      with:
-        path: dist
-        name: ${{ matrix.platform }}-${{ matrix.arch }}-build
-
-
-  sdist-build:
-    name: Build sdist
-    timeout-minutes: 5
-    runs-on: ubuntu-latest
-    strategy:
-      fail-fast: false
-    steps:
-    - uses: actions/checkout@v4
     - name: Set up Python
       uses: actions/setup-python@v5
       with:
         python-version: '3.12'
-    - name: Install dev dependencies
-      run: |
-          python -m pip install --upgrade pip
-          pip install -U -r dev-requirements.txt
-    - name: Create source distribution
-      run: |
-          python setup.py sdist
-    - name: Test sdist
+    - name: Download assets
+      uses: actions/download-artifact@v4
+      with:
+        path: dist
+    - name: Install and test wheel
       shell: bash
       run: |
           rm -rf ./wgpu
-          filename=$(ls dist/*.tar.gz)
-          pip install $filename
-          # don't run tests, we just want to know if the sdist can be installed
+          pip install --find-links dist wgpu
+          pip install --force-reinstall --no-deps --no-index --find-links dist wgpu
+          pushd $HOME
+          python -c 'import wgpu.backends.wgpu_native; print(wgpu.backends.wgpu_native._ffi.lib_path)'
+          popd
           pip uninstall -y wgpu
           git reset --hard HEAD
-    - name: Twine check
-      run: |
-          twine check dist/*
-    - name: Upload distributions
-      uses: actions/upload-artifact@v4
-      with:
-        path: dist
-        name: sdist-build
-
 
   publish:
     name: Publish to Github and Pypi
     runs-on: ubuntu-latest
-    needs: [release-builds, sdist-build]
+    needs: [build-wheels, test-wheels]
     if: success() && startsWith(github.ref, 'refs/tags/v')
     steps:
     - uses: actions/checkout@v4

.github/workflows/ci.yml

@@ -69,7 +69,6 @@ jobs:
       run: |
           python -m pip install --upgrade pip
           pip install requests
-          python download-wgpu-native.py
           pip uninstall -q -y requests
           pip install -e .
     - name: Test imports
@@ -122,7 +121,6 @@ jobs:
       run: |
           python -m pip install --upgrade pip
           pip install -U -r dev-requirements.txt
-          python download-wgpu-native.py
           pip install -e .
     - name: Test examples
       env:
@@ -146,7 +144,6 @@ jobs:
       run: |
           python -m pip install --upgrade pip
           pip install -U requests numpy pytest
-          python download-wgpu-native.py
           pip install -e .
           pip install psutil glfw pyinstaller>=4.9
     - name: Test PyInstaller
@@ -195,7 +192,6 @@ jobs:
       run: |
           python -m pip install --upgrade pip
           pip install -U -r dev-requirements.txt
-          python download-wgpu-native.py
           pip install -e .
     - name: Unit tests
       run: |

pyproject.toml

@@ -1,31 +1,54 @@
+# ===== Project info
+
+[project]
+dynamic = ["version"]
+name = "wgpu"
+description = "WebGPU for Python"
+readme = "README.md"
+license = { file = "LICENSE" }
+authors = [{ name = "Almar Klein" }, { name = "Korijn van Golen" }]
+keywords = ["webgpu", "wgpu", "vulkan", "metal", "DX12", "opengl"]
+requires-python = ">= 3.8"
+dependencies = ["cffi>=1.15.0", "rubicon-objc>=0.4.1; sys_platform == 'darwin'"]
+
+[project.optional-dependencies]
+jupyter = ["jupyter_rfb>=0.4.2"]
+glfw = ["glfw>=1.9"]
+docs = ["sphinx>7.2", "sphinx_rtd_theme"]
+imgui = ["imgui-bundle>=1.2.1"]
+
+[project.entry-points."pyinstaller40"]
+hook-dirs = "wgpu.__pyinstaller:get_hook_dirs"
+tests = "wgpu.__pyinstaller:get_test_dirs"
+
+[project.urls]
+Homepage = "https://github.com/pygfx/wgpu-py"
+Documentation = "https://wgpu-py.readthedocs.io"
+Repository = "https://github.com/pygfx/wgpu-py"
+
+# ===== Building
+#
+# We use the hatch build backend, because its modern, with pretty good defaults,
+# and configurable. Flit is really nice, but a bit too simple for what we need
+# with our binary libs. We use a hatch build hook to install the correct
+# wgpu-native lib right before the wheel is build. See the tools dir.
+
 [build-system]
-requires = ["setuptools>=42"]
-build-backend = "setuptools.build_meta"
-
-
-[tool.cibuildwheel]
-# we only build on one python version since the wheels are not bound to it
-build = "cp312-*"
-# Print system info before build
-before-all = "uname -a"
-# Can't list requests under build-system.requires because that step happens _after_ the before-build command
-before-build = "pip install requests && python download-wgpu-native.py"
-# This is sufficient to trigger an install of the built wheel
-test-command = "echo Wheel installed"
-
-[tool.cibuildwheel.windows]
-# Only for local use, overridden in cd.yml
-archs = ["amd64"]
-
-[tool.cibuildwheel.macos]
-# Only for local use, overridden in cd.yml
-archs = ["arm64"]
-
-[tool.cibuildwheel.linux]
-# wgpu-native does not build for musllinux yet
-skip = "*musllinux*"
-# Use custom images, with minimal version that matches wgpu-native
-manylinux-x86_64-image = "quay.io/pypa/manylinux_2_28_x86_64"
-manylinux-aarch64-image = "quay.io/pypa/manylinux_2_28_aarch64"
-manylinux-i686-image = "quay.io/pypa/manylinux_2_28_i686"
-manylinux-ppc64le-image = "quay.io/pypa/manylinux_2_28_ppc64le"
+requires = ["requests", "hatchling"]
+build-backend = "hatchling.build"
+
+[tool.hatch.version]
+path = "wgpu/__init__.py"
+
+[tool.hatch.build.targets.sdist]
+packages = ["wgpu"]
+exclude = ["*.so", "*.dll", "*.dylib"]
+
+[tool.hatch.build.targets.wheel]
+packages = ["wgpu"]
+artifacts = ["*.so", "*.dll", "*.dylib"]
+
+[tool.hatch.build.targets.wheel.hooks.custom]
+path = "tools/hatch_build.py"
+
+# ===== Tooling

tests/test_set_override.py

@@ -132,7 +132,7 @@ def run_test(
         compute: bool = False,
         vertex_constants=None,
         fragment_constants=None,
-        compute_constants=None
+        compute_constants=None,
     ):
         assert render + compute == 1
         device = self.device