Add Eudonet Paris import GitHub Action

.github/workflows/eudonet_paris_import.yml

@@ -0,0 +1,89 @@
+name: Eudonet Paris Import
+
+# on:
+#   schedule:
+#     - cron: '0 17 * * 1' # Tous les lundis à 17h00
+
+# Temporary
+on:
+  push:
+    branches:
+      - feat/eudonet-auto
+
+jobs:
+  eudonet_paris_import:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v1
+
+      - name: Setup PHP with PECL extension
+        uses: shivammathur/setup-php@v2
+        with:
+          php-version: '8.2'
+
+      - name: Get Composer Cache Directory
+        id: composer-cache
+        run: |
+          echo "dir=$(composer config cache-files-dir)" >> $GITHUB_OUTPUT
+
+      - uses: actions/cache@v3
+        with:
+          path: ${{ steps.composer-cache.outputs.dir }}
+          key: ${{ runner.os }}-composer-${{ hashFiles('**/composer.lock') }}
+          restore-keys: |
+            ${{ runner.os }}-composer-
+
+      - name: Install Scalingo CLI
+        run: curl -O https://cli-dl.scalingo.com/install && bash install
+
+      - name: Install SSH key
+        # Credit: https://stackoverflow.com/a/69234389
+        run: |
+          install -m 600 -D /dev/null ~/.ssh/id_rsa
+          echo "${{ secrets.GH_SCALINGO_SSH_PRIVATE_KEY }}" > ~/.ssh/id_rsa
+
+      - uses: actions/cache@v3
+        id: addok-bundle-cache
+        with:
+          path: docker/addok/addok-data
+          key: ${{ runner.os }}-addok-bundle-
+
+      - name: Download and unzip Addok bundle
+        if: steps.addok-bundle-cache.outputs.cache-hit != 'true'
+        run: |
+          mkdir -p tmp
+          bash tools/download_addok_bundle.sh tmp/addok-archive.zip
+          unzip -d tmp/addok-archive tmp/addok-archive.zip
+          unzip -d docker/addok/addok-data tmp/addok-archive/fichiers/addok-dialog-bundle.zip
+        env:
+          EUDONET_PARIS_KDRIVE_TOKEN: ${{ secrets.EUDONET_PARIS_KDRIVE_TOKEN }}
+
+      - name: Start Addok
+        run: |
+          make addok_start
+          ./tools/wait_for_url.py --interval 5 --max-attempts 20 "http://localhost:7878/search?q=rue+de+la+concertation"
+
+      - name: Init environment variables
+        run: |
+          echo "DATABASE_URL=${{ secrets.EUDONET_PARIS_IMPORT_DATABASE_URL_PR }}" >> .env.local
+          # Deal with JSON quotes
+          printf "APP_EUDONET_PARIS_CREDENTIALS='%s'\n" '${{ secrets.APP_EUDONET_PARIS_CREDENTIALS }}' >> .env.local
+          echo "APP_EUDONET_PARIS_ORG_ID=${{ secrets.APP_EUDONET_PARIS_ORG_ID_PR }}" >> .env.local
+          echo "API_ADRESSE_BASE_URL=http://addok:7878" >> .env.local
+
+      - name: Run import
+        run: make eudonet_paris_import_ci BIN_PHP="php" BIN_CONSOLE="php bin/console" BIN_COMPOSER="composer"
+
+      - name: Get log file path
+        id: logfile
+        if: ${{ !cancelled() }}
+        run:
+          echo "path=$(find log/eudonet_paris -type f -name '*.log' | head -n 1)" >> $GITHUB_OUTPUT
+
+      - uses: actions/upload-artifact@v3
+        if: ${{ !cancelled() }}
+        with:
+          name: eudonet_paris_logfile
+          path:  ${{ steps.logfile.outputs.path }}
+          retention-days: 21

Makefile

@@ -268,3 +268,9 @@ scalingo-node-postbuild:
 scalingo-postdeploy:
 	@echo 'Executing migrations...'
 	${BIN_CONSOLE} doctrine:migrations:migrate --no-interaction
+
+eudonet_paris_import_ci:
+	make composer CMD="install -n --prefer-dist"
+	scalingo login --ssh --ssh-identity ~/.ssh/id_rsa
+	scalingo --app dialog-staging-pr634 db-tunnel -p 10000 DATABASE_URL & ./tools/wait-for-it.sh 127.0.0.1:10000
+	make console CMD="app:eudonet_paris:import"

docs/tools/eudonet_paris.md

@@ -62,3 +62,41 @@ Notes :
 5. Après l'exécution :
   * Vérifiez l'exécution en inspectant le fichier `import.prod-*.log` alimenté pendant l'import.
   * Commentez les variables dans `.env.prod.local` pour éviter de les réutiliser par mégarde jusqu'au prochain import.
+
+## Déploiement périodique automatique
+
+Les données Eudonet Paris sont automatiquement intégrées en production tous les lundis à 17h00.
+
+Cette automatisation est réalisée au moyen de GitHub Actions (voir [`eudonet_paris_import.yml`](../../workflows/eudonet_paris_import.yml)).
+
+### Accès SSH de GitHub Actions à la base de données sur Scalingo
+
+Cette GitHub Action a besoin d'un accès SSH à la base de données hébergée chez Scalingo.
+
+Pour cela des clés SSH ont été générées comme suit :
+
+```bash
+ssh-keygen -t ed25519 -q -N "" -f ~/.ssh/id_dialog_gh_scalingo
+```
+
+La clé publique `~/.ssh/id_dialog_gh_scalingo.pub` ainsi générée a été enregistrée sur Scalingo dans la section [Mes clés SSH](https://dashboard.scalingo.com/account/keys) du compte Scalingo professionnel de @florimondmanca.
+
+> 💡 Pour renouveler les clés, ou en cas de perte, de nouvelles clés peuvent être régénérées en utilisant la méthode ci-dessus, puis rattachées au compte de toute personne ayant un accès "Collaborator" sur l'app Scalingo `dialog`.
+
+La clé privée a été ajoutée comme secret `$GH_SCALINGO_SSH_PRIVATE_KEY` au dépôt GitHub et est utilisée par la GitHub Action.
+
+L'accès à la base de données lors de l'import se fait via un [tunnel chiffré Scalingo](https://doc.scalingo.com/platform/databases/access#encrypted-tunnel).
+
+* L'URL de base de données résultant a été ajouté comme secret `$EUDONET_PARIS_IMPORT_DATABASE_URL`.
+* La valeur de ce secret doit être la `DATABASE_URL` de production où l'on remplace le `host:port` par `127.0.0.1:10000` afin de pointer sur le DB tunnel Scalingo (le port `10000` est hardcodé dans la GitHub Action).
+
+### Données Addok
+
+L'intégration Eudonet Paris a besoin de faire tourner l'[instance Addok personnalisée](./addok.md) en local.
+
+Il faut donc que la GitHub Action télécharge le fichier ZIP contenant les données (1.6 Go environ) hébergé sur le kDrive de Fairness.
+
+Cela est fait par le script `tools/download_addok_bundle.sh`. Pour cela une clé d'API Infomaniak a été créée par @florimondmanca et enregistrée dans le secret `EUDONET_PARIS_KDRIVE_TOKEN`.
+
+Le ZIP est mis en cache après le premier téléchargement.
+

tools/download_addok_bundle.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+set -eux
+
+DRIVE_ID=184671
+FILE_ID=26732
+
+ARCHIVE_ID=$(
+    curl -L \
+        -X POST \
+        -H "Authorization: Bearer ${EUDONET_PARIS_KDRIVE_TOKEN}" \
+        -H "Content-Type: application/json" \
+        -d "{\"file_ids\": [\"${FILE_ID}\"]}" \
+        "https://api.infomaniak.com/3/drive/${DRIVE_ID}/files/archives" \
+        | jq --raw-output .data.uuid
+)
+
+curl -L \
+    -H "Authorization: Bearer ${EUDONET_PARIS_KDRIVE_TOKEN}" \
+    "https://api.infomaniak.com/2/drive/${DRIVE_ID}/files/archives/${ARCHIVE_ID}" \
+    > $1

tools/wait-for-it.sh

@@ -0,0 +1,183 @@
+#!/usr/bin/env bash
+# Credit: https://github.com/vishnubob/wait-for-it
+# Use this script to test if a given TCP host/port are available
+
+WAITFORIT_cmdname=${0##*/}
+
+echoerr() { if [[ $WAITFORIT_QUIET -ne 1 ]]; then echo "$@" 1>&2; fi }
+
+usage()
+{
+    cat << USAGE >&2
+Usage:
+    $WAITFORIT_cmdname host:port [-s] [-t timeout] [-- command args]
+    -h HOST | --host=HOST       Host or IP under test
+    -p PORT | --port=PORT       TCP port under test
+                                Alternatively, you specify the host and port as host:port
+    -s | --strict               Only execute subcommand if the test succeeds
+    -q | --quiet                Don't output any status messages
+    -t TIMEOUT | --timeout=TIMEOUT
+                                Timeout in seconds, zero for no timeout
+    -- COMMAND ARGS             Execute command with args after the test finishes
+USAGE
+    exit 1
+}
+
+wait_for()
+{
+    if [[ $WAITFORIT_TIMEOUT -gt 0 ]]; then
+        echoerr "$WAITFORIT_cmdname: waiting $WAITFORIT_TIMEOUT seconds for $WAITFORIT_HOST:$WAITFORIT_PORT"
+    else
+        echoerr "$WAITFORIT_cmdname: waiting for $WAITFORIT_HOST:$WAITFORIT_PORT without a timeout"
+    fi
+    WAITFORIT_start_ts=$(date +%s)
+    while :
+    do
+        if [[ $WAITFORIT_ISBUSY -eq 1 ]]; then
+            nc -z $WAITFORIT_HOST $WAITFORIT_PORT
+            WAITFORIT_result=$?
+        else
+            (echo -n > /dev/tcp/$WAITFORIT_HOST/$WAITFORIT_PORT) >/dev/null 2>&1
+            WAITFORIT_result=$?
+        fi
+        if [[ $WAITFORIT_result -eq 0 ]]; then
+            WAITFORIT_end_ts=$(date +%s)
+            echoerr "$WAITFORIT_cmdname: $WAITFORIT_HOST:$WAITFORIT_PORT is available after $((WAITFORIT_end_ts - WAITFORIT_start_ts)) seconds"
+            break
+        fi
+        sleep 1
+    done
+    return $WAITFORIT_result
+}
+
+wait_for_wrapper()
+{
+    # In order to support SIGINT during timeout: http://unix.stackexchange.com/a/57692
+    if [[ $WAITFORIT_QUIET -eq 1 ]]; then
+        timeout $WAITFORIT_BUSYTIMEFLAG $WAITFORIT_TIMEOUT $0 --quiet --child --host=$WAITFORIT_HOST --port=$WAITFORIT_PORT --timeout=$WAITFORIT_TIMEOUT &
+    else
+        timeout $WAITFORIT_BUSYTIMEFLAG $WAITFORIT_TIMEOUT $0 --child --host=$WAITFORIT_HOST --port=$WAITFORIT_PORT --timeout=$WAITFORIT_TIMEOUT &
+    fi
+    WAITFORIT_PID=$!
+    trap "kill -INT -$WAITFORIT_PID" INT
+    wait $WAITFORIT_PID
+    WAITFORIT_RESULT=$?
+    if [[ $WAITFORIT_RESULT -ne 0 ]]; then
+        echoerr "$WAITFORIT_cmdname: timeout occurred after waiting $WAITFORIT_TIMEOUT seconds for $WAITFORIT_HOST:$WAITFORIT_PORT"
+    fi
+    return $WAITFORIT_RESULT
+}
+
+# process arguments
+while [[ $# -gt 0 ]]
+do
+    case "$1" in
+        *:* )
+        WAITFORIT_hostport=(${1//:/ })
+        WAITFORIT_HOST=${WAITFORIT_hostport[0]}
+        WAITFORIT_PORT=${WAITFORIT_hostport[1]}
+        shift 1
+        ;;
+        --child)
+        WAITFORIT_CHILD=1
+        shift 1
+        ;;
+        -q | --quiet)
+        WAITFORIT_QUIET=1
+        shift 1
+        ;;
+        -s | --strict)
+        WAITFORIT_STRICT=1
+        shift 1
+        ;;
+        -h)
+        WAITFORIT_HOST="$2"
+        if [[ $WAITFORIT_HOST == "" ]]; then break; fi
+        shift 2
+        ;;
+        --host=*)
+        WAITFORIT_HOST="${1#*=}"
+        shift 1
+        ;;
+        -p)
+        WAITFORIT_PORT="$2"
+        if [[ $WAITFORIT_PORT == "" ]]; then break; fi
+        shift 2
+        ;;
+        --port=*)
+        WAITFORIT_PORT="${1#*=}"
+        shift 1
+        ;;
+        -t)
+        WAITFORIT_TIMEOUT="$2"
+        if [[ $WAITFORIT_TIMEOUT == "" ]]; then break; fi
+        shift 2
+        ;;
+        --timeout=*)
+        WAITFORIT_TIMEOUT="${1#*=}"
+        shift 1
+        ;;
+        --)
+        shift
+        WAITFORIT_CLI=("$@")
+        break
+        ;;
+        --help)
+        usage
+        ;;
+        *)
+        echoerr "Unknown argument: $1"
+        usage
+        ;;
+    esac
+done
+
+if [[ "$WAITFORIT_HOST" == "" || "$WAITFORIT_PORT" == "" ]]; then
+    echoerr "Error: you need to provide a host and port to test."
+    usage
+fi
+
+WAITFORIT_TIMEOUT=${WAITFORIT_TIMEOUT:-15}
+WAITFORIT_STRICT=${WAITFORIT_STRICT:-0}
+WAITFORIT_CHILD=${WAITFORIT_CHILD:-0}
+WAITFORIT_QUIET=${WAITFORIT_QUIET:-0}
+
+# Check to see if timeout is from busybox?
+WAITFORIT_TIMEOUT_PATH=$(type -p timeout)
+WAITFORIT_TIMEOUT_PATH=$(realpath $WAITFORIT_TIMEOUT_PATH 2>/dev/null || readlink -f $WAITFORIT_TIMEOUT_PATH)
+
+WAITFORIT_BUSYTIMEFLAG=""
+if [[ $WAITFORIT_TIMEOUT_PATH =~ "busybox" ]]; then
+    WAITFORIT_ISBUSY=1
+    # Check if busybox timeout uses -t flag
+    # (recent Alpine versions don't support -t anymore)
+    if timeout &>/dev/stdout | grep -q -e '-t '; then
+        WAITFORIT_BUSYTIMEFLAG="-t"
+    fi
+else
+    WAITFORIT_ISBUSY=0
+fi
+
+if [[ $WAITFORIT_CHILD -gt 0 ]]; then
+    wait_for
+    WAITFORIT_RESULT=$?
+    exit $WAITFORIT_RESULT
+else
+    if [[ $WAITFORIT_TIMEOUT -gt 0 ]]; then
+        wait_for_wrapper
+        WAITFORIT_RESULT=$?
+    else
+        wait_for
+        WAITFORIT_RESULT=$?
+    fi
+fi
+
+if [[ $WAITFORIT_CLI != "" ]]; then
+    if [[ $WAITFORIT_RESULT -ne 0 && $WAITFORIT_STRICT -eq 1 ]]; then
+        echoerr "$WAITFORIT_cmdname: strict mode, refusing to execute subprocess"
+        exit $WAITFORIT_RESULT
+    fi
+    exec "${WAITFORIT_CLI[@]}"
+else
+    exit $WAITFORIT_RESULT
+fi

tools/wait_for_url.py

@@ -0,0 +1,40 @@
+#!/usr/bin/env python3
+# Use this script to wait for a given URL to become available
+import argparse
+import subprocess
+import time
+import sys
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("url")
+    parser.add_argument("--interval", type=int, default=3)
+    parser.add_argument("--max-attempts", type=int, default=5)
+    args = parser.parse_args()
+
+    url = args.url
+    interval = args.interval
+    max_attempts = args.max_attempts
+
+    start_time = time.time()
+
+    print(f"wait-for-url.py: waiting {interval * max_attempts} seconds for {url}")
+
+    for _ in range(max_attempts):
+        result = subprocess.run(
+            ["curl", "--output", "/dev/null", "--silent", "--fail", url]
+        )
+
+        if result.returncode == 0:
+            elapsed = time.time() - start_time
+            print(f"wait-for-url.py: {url} is available after {elapsed:.0f} seconds")
+            break
+
+        print(".", end="", flush=True)
+        time.sleep(interval)
+    else:
+        print()
+        print(f"wait-for-it.py: {url} failed to become available")
+        sys.exit(1)
+
+    sys.exit(0)