Optimize scraping #8

.github/workflows/release.yml

@@ -3,8 +3,18 @@ on:
   push:
     branches:
       - main
+    paths:
+      - 'src/**'
+      - 'custom/**'
   schedule:
     - cron: '0 0 1 * *'
+  workflow_dispatch:
+    inputs:
+      fullScrape:
+        description: 'Scrape the entire wiki, instead of only the changed pages'
+        required: true
+        default: false
+        type: boolean
 jobs:
   release:
     permissions: write-all
@@ -32,7 +42,14 @@ jobs:
         uses: montudor/action-zip@v1
       - name: Scrape wiki
         if: ${{ steps.wiki_confirm_no_changes.outcome == 'failure' }}
-        run: npm run scrape-wiki
+        env:
+          FULL_SCRAPE: ${{ inputs.fullScrape || false }}
+        run: |
+          if [ "$FULL_SCRAPE" = "true" ]; then
+            npm run wiki-scrape
+          else
+            npm run wiki-scrape -- --update-changed-by-tag $(git tag -l --sort=-v:refname | head -n 1)
+          fi
       - name: Format the output with StyLua
         if: ${{ steps.wiki_confirm_no_changes.outcome == 'failure' }}
         uses: JohnnyMorganz/[email protected]

package.json

@@ -34,6 +34,7 @@
     "archiver": "^6.0.1",
     "cheerio": "^1.0.0-rc.12",
     "cross-env": "^7.0.3",
+    "extract-zip": "^2.0.1",
     "fetch-retry": "^5.0.6",
     "html-loader": "^4.2.0",
     "jest": "^29.7.0",