diff --git a/.github/workflows/pytests.yml b/.github/workflows/pytests.yml
index a84032bc..7d060151 100644
--- a/.github/workflows/pytests.yml
+++ b/.github/workflows/pytests.yml
@@ -38,7 +38,7 @@ jobs:
           python3 --version
 
       - name: Install dependencies from pip
-        run: python3 -m pip install wheel setuptools numpy scipy click matplotlib pandas pyyaml spglib rdkit flake8 pytest pytest-cov requests
+        run: python3 -m pip install wheel setuptools numpy scipy click matplotlib pyyaml spglib rdkit flake8 pytest pytest-cov requests
 
       - name: Check numpy
         run: |
@@ -93,6 +93,12 @@ jobs:
       - name: Install Quippy from PyPI
         run: python3 -m pip install quippy-ase
 
+      - name: Install xTB (before things that need pandas like MACE and wfl, since it will break pandas-numpy compatibility by downgrading numpy)
+        run: |
+          conda install -c conda-forge xtb-python
+          python3 -m pip install typing-extensions
+          python3 -m pip install pandas
+
       - name: MACE
         run: |
           echo "search for torch version"
@@ -143,11 +149,6 @@ jobs:
         run: |
           echo $HOME/bin >> $GITHUB_PATH
 
-      - name: Install xTB
-        run: |
-          conda install -c conda-forge xtb-python
-          python3 -m pip install typing-extensions
-
       - name: Install MPI dependencies
         if: env.use-mpi
         run: |