Import/Update from Github Actions

hiddewie · Mar 30, 2024 · 3feb113 · 3feb113
1 parent d7399ea
commit 3feb113
Show file tree

Hide file tree

Showing 4 changed files with 219 additions and 109 deletions.
diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml
@@ -1,15 +1,15 @@
 name: Deploy
+
 on:
   workflow_dispatch: ~
 
-env:
-  FLY_API_TOKEN: ${{ secrets.FLY_API_TOKEN }}
-
 jobs:
   deploy-proxy:
     name: Deploy proxy
     runs-on: ubuntu-latest
     environment: proxy-production
+
+
     steps:
       - name: Checkout
         uses: actions/checkout@v4
@@ -21,51 +21,64 @@ jobs:
         uses: superfly/flyctl-actions/setup-flyctl@master
 
       - name: Deploy
+        env:
+          FLY_API_TOKEN: ${{ secrets.FLY_API_TOKEN }}
         run: |
           flyctl deploy --config proxy.fly.toml --local-only
 
-  deploy-tiles:
-    name: Deploy tiles
+# Import steps
+  #      - name: Install Osmium
+  #        run: |
+  #          sudo apt install osmium-tool
+
+  #      - name: Generate cache key
+  #        id: cache-key
+  #        run: |
+  #          echo "key=$(date +%Y-%W)" >> $GITHUB_OUTPUT
+
+  #      - name: Cache downloaded OpenStreetMap data
+  #        id: cache-data
+  #        uses: actions/cache@v4
+  #        with:
+  #          key: 'data-europe-${{ steps.cache-key.outputs.key }}'
+  #          path: data/filtered/europe.osm.pbf
+  #
+  #      - name: Download data
+  #        if: ${{ steps.cache-data.outputs.cache-hit != 'true' }}
+  #        run: |
+  #          curl --progress-bar http://ftp.snt.utwente.nl/pub/misc/openstreetmap/europe-latest.osm.pbf --output data/europe.osm.pbf
+
+#      - name: Import data
+#        env:
+#          # Available cores: 4
+#          OSM2PGSQL_NUMPROC: 4
+#          # Available memory: 16GB
+#          OSM2PGSQL_CACHE: 14000
+#          OSM2PGSQL_DATAFILE: europe.osm.pbf
+#        run: |
+#          docker compose pull import
+#          docker compose build import
+#          docker compose run import import
+#          docker compose push import
+
+  update-data:
+    name: Update data
     runs-on: ubuntu-latest
-    environment: tiles-production
+
     steps:
       - name: Checkout
         uses: actions/checkout@v4
 
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v3
 
-      - name: Setup Fly
-        uses: superfly/flyctl-actions/setup-flyctl@master
-
       - name: Login to GitHub Container Registry
         uses: docker/login-action@v3
         with:
           registry: ghcr.io
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
 
-#      - name: Install Osmium
-#        run: |
-#          sudo apt install osmium-tool
-
-#      - name: Generate cache key
-#        id: cache-key
-#        run: |
-#          echo "key=$(date +%Y-%W)" >> $GITHUB_OUTPUT
-
-#      - name: Cache downloaded OpenStreetMap data
-#        id: cache-data
-#        uses: actions/cache@v4
-#        with:
-#          key: 'data-europe-${{ steps.cache-key.outputs.key }}'
-#          path: data/filtered/europe.osm.pbf
-#
-#      - name: Download data
-#        if: ${{ steps.cache-data.outputs.cache-hit != 'true' }}
-#        run: |
-#          curl --progress-bar http://ftp.snt.utwente.nl/pub/misc/openstreetmap/europe-latest.osm.pbf --output data/europe.osm.pbf
-
       # The disk and memory in the Github runners is too small to do large imports.
       # A Docker image with a pre-imported database state is used.
       # During the deployment run, the docker image is updated through the replication updated,
@@ -76,18 +89,16 @@ jobs:
           docker image ls ghcr.io/hiddewie/openrailwaymap-import-db:latest
           docker compose up --no-build --pull never --detach db
 
-#      - name: Import data
       - name: Ensure data is up to date
         env:
           # Available cores: 4
           OSM2PGSQL_NUMPROC: 4
           # Available memory: 16GB
           OSM2PGSQL_CACHE: 14000
-#          OSM2PGSQL_DATAFILE: europe.osm.pbf
         run: |
           docker compose pull import
           docker compose build import
-          docker compose run import
+          docker compose run import update
           docker compose push import
 
       - name: Commit and push new database image
@@ -99,12 +110,90 @@ jobs:
           docker image ls "$DB_IMAGE"
           docker push "$DB_IMAGE"
 
-      - name: Generate tiles
+  generate-tiles:
+    name: Generate tiles
+    runs-on: ubuntu-latest
+    needs: update-data
+
+    strategy:
+      matrix:
+        tiles:
+          - low-med
+          - standard
+          - speed
+          - signals
+          - electrification
+          - gauge
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Pull import database
         run: |
+          docker compose pull db
           docker compose up --no-build --pull never --detach db
-          docker compose build martin-cp
+
+      - name: Generate tiles
+        env:
+          TILES: ${{ matrix.tiles }}
+        run: |
           docker compose run martin-cp
+          ls -las tiles/
+
+      - uses: actions/upload-artifact@v4
+        with:
+          name: tiles-${{ matrix.tiles }}
+          path: |
+            tiles/*.mbtiles
+          if-no-files-found: error
+          retention-days: 3
+
+  deploy-tiles:
+    name: Deploy tiles
+    runs-on: ubuntu-latest
+    needs: generate-tiles
+    environment: tiles-production
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Setup Fly
+        uses: superfly/flyctl-actions/setup-flyctl@master
+
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - uses: actions/download-artifact@v4
+        with:
+          # No name: download everything
+          path: tiles
+          merge-multiple: true
+
+      - name: List generated tiles
+        run: |
+          ls -lah tiles/
 
       - name: Deploy
+        env:
+          FLY_API_TOKEN: ${{ secrets.FLY_API_TOKEN }}
         run: |
           flyctl deploy --config martin-static.fly.toml --local-only
diff --git a/docker-compose.yml b/docker-compose.yml
@@ -40,26 +40,26 @@ services:
 
   martin-cp:
     image: ghcr.io/maplibre/martin
+    depends_on:
+      - db
     entrypoint: ['sh', '-c']
     command: |
       '
       export MARTIN="martin-cp --config /config/configuration.yml --mbtiles-type flat --on-duplicate override"
-      $$MARTIN --min-zoom 0 --max-zoom 6 "--bbox=$$BBOX" --source railway_line_low --output-file /tiles/railway_line_low.mbtiles  && mbtiles summary /tiles/railway_line_low.mbtiles &&
-      $$MARTIN --min-zoom 0 --max-zoom 6 "--bbox=$$BBOX" --source standard_railway_text_stations_low --output-file /tiles/standard_railway_text_stations_low.mbtiles && mbtiles summary /tiles/standard_railway_text_stations_low.mbtiles &&
-      $$MARTIN --min-zoom 7 --max-zoom 7 "--bbox=$$BBOX" --source railway_line_med --output-file /tiles/railway_line_med.mbtiles && mbtiles summary /tiles/railway_line_med.mbtiles &&
-      $$MARTIN --min-zoom 7 --max-zoom 7 "--bbox=$$BBOX" --source standard_railway_text_stations_med --output-file /tiles/standard_railway_text_stations_med.mbtiles && mbtiles summary /tiles/standard_railway_text_stations_med.mbtiles &&
-      $$MARTIN --min-zoom 8 --max-zoom "$$MAX_ZOOM" "--bbox=$$BBOX" --source standard_railway_turntables,standard_railway_line_fill,standard_railway_text_stations,standard_railway_symbols,standard_railway_text_km,standard_railway_switch_ref --output-file /tiles/standard.mbtiles && mbtiles summary /tiles/standard.mbtiles &&
-      $$MARTIN --min-zoom 8 --max-zoom "$$MAX_ZOOM" "--bbox=$$BBOX" --source speed_railway_line_casing,speed_railway_line_fill,speed_railway_signals --output-file /tiles/speed.mbtiles && mbtiles summary /tiles/speed.mbtiles &&
-      $$MARTIN --min-zoom 8 --max-zoom "$$MAX_ZOOM" "--bbox=$$BBOX" --source signals_railway_line,signals_railway_signals,signals_signal_boxes --output-file /tiles/signals.mbtiles && mbtiles summary /tiles/signals.mbtiles &&
-      $$MARTIN --min-zoom 8 --max-zoom "$$MAX_ZOOM" "--bbox=$$BBOX" --source electrification_railway_line,electrification_future,electrification_signals --output-file /tiles/electrification.mbtiles && mbtiles summary /tiles/electrification.mbtiles &&
-      $$MARTIN --min-zoom 8 --max-zoom "$$MAX_ZOOM" "--bbox=$$BBOX" --source gauge_railway_line_low,gauge_railway_line --output-file /tiles/gauge.mbtiles && mbtiles summary /tiles/gauge.mbtiles
+      [ "$$TILES" = "low-med" ] && $$MARTIN --min-zoom 0 --max-zoom 6 "--bbox=$$BBOX" --source railway_line_low --output-file /tiles/railway_line_low.mbtiles  && mbtiles summary /tiles/railway_line_low.mbtiles
+      [ "$$TILES" = "low-med" ] && $$MARTIN --min-zoom 0 --max-zoom 6 "--bbox=$$BBOX" --source standard_railway_text_stations_low --output-file /tiles/standard_railway_text_stations_low.mbtiles && mbtiles summary /tiles/standard_railway_text_stations_low.mbtiles
+      [ "$$TILES" = "low-med" ] && $$MARTIN --min-zoom 7 --max-zoom 7 "--bbox=$$BBOX" --source railway_line_med --output-file /tiles/railway_line_med.mbtiles && mbtiles summary /tiles/railway_line_med.mbtiles
+      [ "$$TILES" = "low-med" ] && $$MARTIN --min-zoom 7 --max-zoom 7 "--bbox=$$BBOX" --source standard_railway_text_stations_med --output-file /tiles/standard_railway_text_stations_med.mbtiles && mbtiles summary /tiles/standard_railway_text_stations_med.mbtiles
+      [ "$$TILES" = "standard" ] && $$MARTIN --min-zoom 8 --max-zoom "$$MAX_ZOOM" "--bbox=$$BBOX" --source standard_railway_turntables,standard_railway_line_fill,standard_railway_text_stations,standard_railway_symbols,standard_railway_text_km,standard_railway_switch_ref --output-file /tiles/standard.mbtiles && mbtiles summary /tiles/standard.mbtiles
+      [ "$$TILES" = "speed" ] && $$MARTIN --min-zoom 8 --max-zoom "$$MAX_ZOOM" "--bbox=$$BBOX" --source speed_railway_line_casing,speed_railway_line_fill,speed_railway_signals --output-file /tiles/speed.mbtiles && mbtiles summary /tiles/speed.mbtiles
+      [ "$$TILES" = "signals" ] && $$MARTIN --min-zoom 8 --max-zoom "$$MAX_ZOOM" "--bbox=$$BBOX" --source signals_railway_line,signals_railway_signals,signals_signal_boxes --output-file /tiles/signals.mbtiles && mbtiles summary /tiles/signals.mbtiles
+      [ "$$TILES" = "electrification" ] && $$MARTIN --min-zoom 8 --max-zoom "$$MAX_ZOOM" "--bbox=$$BBOX" --source electrification_railway_line,electrification_future,electrification_signals --output-file /tiles/electrification.mbtiles && mbtiles summary /tiles/electrification.mbtiles
+      [ "$$TILES" = "gauge" ] && $$MARTIN --min-zoom 8 --max-zoom "$$MAX_ZOOM" "--bbox=$$BBOX" --source gauge_railway_line_low,gauge_railway_line --output-file /tiles/gauge.mbtiles && mbtiles summary /tiles/gauge.mbtiles
       '
 
     volumes:
       - ./martin:/config
       - ./tiles:/tiles
-    depends_on:
-      - db
     environment:
       # Europe center: -11.3818,35.8891,25.0488,57.3976
       # AT: --bbox=9.52678,46.36851,17.16273,48.90201
@@ -70,6 +70,7 @@ services:
       - MIN_ZOOM=0
       - MAX_ZOOM=14
       - DATABASE_URL=postgresql://postgres@db:5432/gis
+      - TILES
 
   martin:
     build:

diff --git a/import.Dockerfile b/import.Dockerfile
@@ -26,4 +26,7 @@ COPY import/sql sql
 COPY import/openrailwaymap.lua openrailwaymap.lua
 COPY import/docker-startup.sh docker-startup.sh
 
-CMD ["/openrailwaymap/docker-startup.sh"]
+ENTRYPOINT ["/openrailwaymap/docker-startup.sh"]
+
+# Empty command for adding arguments to the startup script
+CMD []