diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml
index e379407b..c324fd60 100644
--- a/.github/workflows/docs.yml
+++ b/.github/workflows/docs.yml
@@ -60,12 +60,14 @@ jobs:
         run: |
           mkdir doc_cache  # make sure cache dir exists
 
-      - name: Cache FastF1
-        uses: actions/cache@v4
+      - name: Restore cache for FastF1
+        id: cache-doc-restore
+        uses: actions/cache/restore@v4
         with:
-          save-always: true
           path: ./doc_cache
-          key: fastf1-doc-cache-${{ hashFiles('*.*') }}
+           # Restore any matching cache independent of hash suffix. We cannot
+           # know if anything has changed at this stage.
+          key: fastf1-doc-cache-${{ hashFiles('./doc_cache/**/*.*') }}
           restore-keys: |
             fastf1-doc-cache
 
@@ -94,6 +96,15 @@ jobs:
         with:
           path: docs/_build/html/
 
+      - name: Save cache for FastF1
+        id: cache-doc-save
+        # Save the cache even in case of a failure but only if the content of
+        # the cached directory has changed.
+        if: always() && steps.cache-doc-restore.outputs.cache-matched-key != hashFiles('./doc_cache/**/*.*')
+        uses: actions/cache/save@v4
+        with:
+          key: fastf1-doc-cache-${{ hashFiles('./doc_cache/**/*.*') }}
+          path: ./doc_cache
 
   deploy:
     if: (github.event_name == 'release') || inputs.publish
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
index 14e6b22e..008e467f 100644
--- a/.github/workflows/tests.yml
+++ b/.github/workflows/tests.yml
@@ -61,12 +61,14 @@ jobs:
         run: |
           mkdir test_cache  # make sure cache dir exists
 
-      - name: Cache FastF1
+      - name: Restore cache for FastF1
+        id: cache-fastf1-restore
         uses: actions/cache@v4
         with:
-          save-always: true
           path: ./test_cache
-          key: fastf1-${{ matrix.python-version }}${{ matrix.cache-suffix }}-${{ hashFiles('*.*') }}
+          # Restore any matching cache independent of hash suffix. We cannot
+          # know if anything has changed at this stage.
+          key: fastf1-${{ matrix.python-version }}${{ matrix.cache-suffix }}-${{ hashFiles('./test_cache/**/*.*') }}
           restore-keys: |
             fastf1-${{ matrix.python-version }}${{ matrix.cache-suffix }}
 
@@ -74,6 +76,16 @@ jobs:
         run: |
           pytest -ra
 
+      - name: Save cache for FastF1
+        id: cache-fastf1-save
+        # Save the cache even in case of a failure but only if the content of
+        # the cached directory has changed.
+        if: always() && steps.cache-fastf1-restore.outputs.cache-matched-key != hashFiles('./test_cache/**/*.*')
+        uses: actions/cache/save@v4
+        with:
+          key: fastf1-${{ matrix.python-version }}${{ matrix.cache-suffix }}-${{ hashFiles('./test_cache/**/*.*') }}
+          path: ./test_cache
+
 
   run-lint-checks:
     if: (!contains(github.event.head_commit.message, '[skip-ruff]'))