diff --git a/.github/workflows/eudonet_paris_import.yml b/.github/workflows/eudonet_paris_import.yml new file mode 100644 index 000000000..a05d6134d --- /dev/null +++ b/.github/workflows/eudonet_paris_import.yml @@ -0,0 +1,89 @@ +name: Eudonet Paris Import + +# on: +# schedule: +# - cron: '0 17 * * 1' # Tous les lundis à 17h00 + +# Temporary +on: + push: + branches: + - feat/eudonet-auto + +jobs: + eudonet_paris_import: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v1 + + - name: Setup PHP with PECL extension + uses: shivammathur/setup-php@v2 + with: + php-version: '8.2' + + - name: Get Composer Cache Directory + id: composer-cache + run: | + echo "dir=$(composer config cache-files-dir)" >> $GITHUB_OUTPUT + + - uses: actions/cache@v3 + with: + path: ${{ steps.composer-cache.outputs.dir }} + key: ${{ runner.os }}-composer-${{ hashFiles('**/composer.lock') }} + restore-keys: | + ${{ runner.os }}-composer- + + - name: Install Scalingo CLI + run: curl -O https://cli-dl.scalingo.com/install && bash install + + - name: Install SSH key + # Credit: https://stackoverflow.com/a/69234389 + run: | + install -m 600 -D /dev/null ~/.ssh/id_rsa + echo "${{ secrets.GH_SCALINGO_SSH_PRIVATE_KEY }}" > ~/.ssh/id_rsa + + - uses: actions/cache@v3 + id: addok-bundle-cache + with: + path: docker/addok/addok-data + key: ${{ runner.os }}-addok-bundle- + + - name: Download and unzip Addok bundle + if: steps.addok-bundle-cache.outputs.cache-hit != 'true' + run: | + mkdir -p tmp + bash tools/download_addok_bundle.sh tmp/addok-archive.zip + unzip -d tmp/addok-archive tmp/addok-archive.zip + unzip -d docker/addok/addok-data tmp/addok-archive/fichiers/addok-dialog-bundle.zip + env: + EUDONET_PARIS_KDRIVE_TOKEN: ${{ secrets.EUDONET_PARIS_KDRIVE_TOKEN }} + + - name: Start Addok + run: | + make addok_start + ./tools/wait_for_url.py --interval 5 --max-attempts 20 "http://localhost:7878/search?q=rue+de+la+concertation" + + - name: Init environment variables + run: | + echo "DATABASE_URL=${{ secrets.EUDONET_PARIS_IMPORT_DATABASE_URL_PR }}" >> .env.local + # Deal with JSON quotes + printf "APP_EUDONET_PARIS_CREDENTIALS='%s'\n" '${{ secrets.APP_EUDONET_PARIS_CREDENTIALS }}' >> .env.local + echo "APP_EUDONET_PARIS_ORG_ID=${{ secrets.APP_EUDONET_PARIS_ORG_ID_PR }}" >> .env.local + echo "API_ADRESSE_BASE_URL=http://addok:7878" >> .env.local + + - name: Run import + run: make eudonet_paris_import_ci BIN_PHP="php" BIN_CONSOLE="php bin/console" BIN_COMPOSER="composer" + + - name: Get log file path + id: logfile + if: ${{ !cancelled() }} + run: + echo "path=$(find log/eudonet_paris -type f -name '*.log' | head -n 1)" >> $GITHUB_OUTPUT + + - uses: actions/upload-artifact@v3 + if: ${{ !cancelled() }} + with: + name: eudonet_paris_logfile + path: ${{ steps.logfile.outputs.path }} + retention-days: 21 diff --git a/Makefile b/Makefile index 411cbf1a5..9cb7c64bf 100644 --- a/Makefile +++ b/Makefile @@ -268,3 +268,9 @@ scalingo-node-postbuild: scalingo-postdeploy: @echo 'Executing migrations...' ${BIN_CONSOLE} doctrine:migrations:migrate --no-interaction + +eudonet_paris_import_ci: + make composer CMD="install -n --prefer-dist" + scalingo login --ssh --ssh-identity ~/.ssh/id_rsa + scalingo --app dialog-staging-pr634 db-tunnel -p 10000 DATABASE_URL & ./tools/wait-for-it.sh 127.0.0.1:10000 + make console CMD="app:eudonet_paris:import" diff --git a/docs/tools/eudonet_paris.md b/docs/tools/eudonet_paris.md index 6e5944b17..9b412fe9d 100644 --- a/docs/tools/eudonet_paris.md +++ b/docs/tools/eudonet_paris.md @@ -62,3 +62,41 @@ Notes : 5. Après l'exécution : * Vérifiez l'exécution en inspectant le fichier `import.prod-*.log` alimenté pendant l'import. * Commentez les variables dans `.env.prod.local` pour éviter de les réutiliser par mégarde jusqu'au prochain import. + +## Déploiement périodique automatique + +Les données Eudonet Paris sont automatiquement intégrées en production tous les lundis à 17h00. + +Cette automatisation est réalisée au moyen de GitHub Actions (voir [`eudonet_paris_import.yml`](../../workflows/eudonet_paris_import.yml)). + +### Accès SSH de GitHub Actions à la base de données sur Scalingo + +Cette GitHub Action a besoin d'un accès SSH à la base de données hébergée chez Scalingo. + +Pour cela des clés SSH ont été générées comme suit : + +```bash +ssh-keygen -t ed25519 -q -N "" -f ~/.ssh/id_dialog_gh_scalingo +``` + +La clé publique `~/.ssh/id_dialog_gh_scalingo.pub` ainsi générée a été enregistrée sur Scalingo dans la section [Mes clés SSH](https://dashboard.scalingo.com/account/keys) du compte Scalingo professionnel de @florimondmanca. + +> 💡 Pour renouveler les clés, ou en cas de perte, de nouvelles clés peuvent être régénérées en utilisant la méthode ci-dessus, puis rattachées au compte de toute personne ayant un accès "Collaborator" sur l'app Scalingo `dialog`. + +La clé privée a été ajoutée comme secret `$GH_SCALINGO_SSH_PRIVATE_KEY` au dépôt GitHub et est utilisée par la GitHub Action. + +L'accès à la base de données lors de l'import se fait via un [tunnel chiffré Scalingo](https://doc.scalingo.com/platform/databases/access#encrypted-tunnel). + +* L'URL de base de données résultant a été ajouté comme secret `$EUDONET_PARIS_IMPORT_DATABASE_URL`. +* La valeur de ce secret doit être la `DATABASE_URL` de production où l'on remplace le `host:port` par `127.0.0.1:10000` afin de pointer sur le DB tunnel Scalingo (le port `10000` est hardcodé dans la GitHub Action). + +### Données Addok + +L'intégration Eudonet Paris a besoin de faire tourner l'[instance Addok personnalisée](./addok.md) en local. + +Il faut donc que la GitHub Action télécharge le fichier ZIP contenant les données (1.6 Go environ) hébergé sur le kDrive de Fairness. + +Cela est fait par le script `tools/download_addok_bundle.sh`. Pour cela une clé d'API Infomaniak a été créée par @florimondmanca et enregistrée dans le secret `EUDONET_PARIS_KDRIVE_TOKEN`. + +Le ZIP est mis en cache après le premier téléchargement. + diff --git a/tools/download_addok_bundle.sh b/tools/download_addok_bundle.sh new file mode 100755 index 000000000..0ef1eaad1 --- /dev/null +++ b/tools/download_addok_bundle.sh @@ -0,0 +1,20 @@ +#!/bin/bash +set -eux + +DRIVE_ID=184671 +FILE_ID=26732 + +ARCHIVE_ID=$( + curl -L \ + -X POST \ + -H "Authorization: Bearer ${EUDONET_PARIS_KDRIVE_TOKEN}" \ + -H "Content-Type: application/json" \ + -d "{\"file_ids\": [\"${FILE_ID}\"]}" \ + "https://api.infomaniak.com/3/drive/${DRIVE_ID}/files/archives" \ + | jq --raw-output .data.uuid +) + +curl -L \ + -H "Authorization: Bearer ${EUDONET_PARIS_KDRIVE_TOKEN}" \ + "https://api.infomaniak.com/2/drive/${DRIVE_ID}/files/archives/${ARCHIVE_ID}" \ + > $1 diff --git a/tools/wait-for-it.sh b/tools/wait-for-it.sh new file mode 100755 index 000000000..8fa7512b0 --- /dev/null +++ b/tools/wait-for-it.sh @@ -0,0 +1,183 @@ +#!/usr/bin/env bash +# Credit: https://github.com/vishnubob/wait-for-it +# Use this script to test if a given TCP host/port are available + +WAITFORIT_cmdname=${0##*/} + +echoerr() { if [[ $WAITFORIT_QUIET -ne 1 ]]; then echo "$@" 1>&2; fi } + +usage() +{ + cat << USAGE >&2 +Usage: + $WAITFORIT_cmdname host:port [-s] [-t timeout] [-- command args] + -h HOST | --host=HOST Host or IP under test + -p PORT | --port=PORT TCP port under test + Alternatively, you specify the host and port as host:port + -s | --strict Only execute subcommand if the test succeeds + -q | --quiet Don't output any status messages + -t TIMEOUT | --timeout=TIMEOUT + Timeout in seconds, zero for no timeout + -- COMMAND ARGS Execute command with args after the test finishes +USAGE + exit 1 +} + +wait_for() +{ + if [[ $WAITFORIT_TIMEOUT -gt 0 ]]; then + echoerr "$WAITFORIT_cmdname: waiting $WAITFORIT_TIMEOUT seconds for $WAITFORIT_HOST:$WAITFORIT_PORT" + else + echoerr "$WAITFORIT_cmdname: waiting for $WAITFORIT_HOST:$WAITFORIT_PORT without a timeout" + fi + WAITFORIT_start_ts=$(date +%s) + while : + do + if [[ $WAITFORIT_ISBUSY -eq 1 ]]; then + nc -z $WAITFORIT_HOST $WAITFORIT_PORT + WAITFORIT_result=$? + else + (echo -n > /dev/tcp/$WAITFORIT_HOST/$WAITFORIT_PORT) >/dev/null 2>&1 + WAITFORIT_result=$? + fi + if [[ $WAITFORIT_result -eq 0 ]]; then + WAITFORIT_end_ts=$(date +%s) + echoerr "$WAITFORIT_cmdname: $WAITFORIT_HOST:$WAITFORIT_PORT is available after $((WAITFORIT_end_ts - WAITFORIT_start_ts)) seconds" + break + fi + sleep 1 + done + return $WAITFORIT_result +} + +wait_for_wrapper() +{ + # In order to support SIGINT during timeout: http://unix.stackexchange.com/a/57692 + if [[ $WAITFORIT_QUIET -eq 1 ]]; then + timeout $WAITFORIT_BUSYTIMEFLAG $WAITFORIT_TIMEOUT $0 --quiet --child --host=$WAITFORIT_HOST --port=$WAITFORIT_PORT --timeout=$WAITFORIT_TIMEOUT & + else + timeout $WAITFORIT_BUSYTIMEFLAG $WAITFORIT_TIMEOUT $0 --child --host=$WAITFORIT_HOST --port=$WAITFORIT_PORT --timeout=$WAITFORIT_TIMEOUT & + fi + WAITFORIT_PID=$! + trap "kill -INT -$WAITFORIT_PID" INT + wait $WAITFORIT_PID + WAITFORIT_RESULT=$? + if [[ $WAITFORIT_RESULT -ne 0 ]]; then + echoerr "$WAITFORIT_cmdname: timeout occurred after waiting $WAITFORIT_TIMEOUT seconds for $WAITFORIT_HOST:$WAITFORIT_PORT" + fi + return $WAITFORIT_RESULT +} + +# process arguments +while [[ $# -gt 0 ]] +do + case "$1" in + *:* ) + WAITFORIT_hostport=(${1//:/ }) + WAITFORIT_HOST=${WAITFORIT_hostport[0]} + WAITFORIT_PORT=${WAITFORIT_hostport[1]} + shift 1 + ;; + --child) + WAITFORIT_CHILD=1 + shift 1 + ;; + -q | --quiet) + WAITFORIT_QUIET=1 + shift 1 + ;; + -s | --strict) + WAITFORIT_STRICT=1 + shift 1 + ;; + -h) + WAITFORIT_HOST="$2" + if [[ $WAITFORIT_HOST == "" ]]; then break; fi + shift 2 + ;; + --host=*) + WAITFORIT_HOST="${1#*=}" + shift 1 + ;; + -p) + WAITFORIT_PORT="$2" + if [[ $WAITFORIT_PORT == "" ]]; then break; fi + shift 2 + ;; + --port=*) + WAITFORIT_PORT="${1#*=}" + shift 1 + ;; + -t) + WAITFORIT_TIMEOUT="$2" + if [[ $WAITFORIT_TIMEOUT == "" ]]; then break; fi + shift 2 + ;; + --timeout=*) + WAITFORIT_TIMEOUT="${1#*=}" + shift 1 + ;; + --) + shift + WAITFORIT_CLI=("$@") + break + ;; + --help) + usage + ;; + *) + echoerr "Unknown argument: $1" + usage + ;; + esac +done + +if [[ "$WAITFORIT_HOST" == "" || "$WAITFORIT_PORT" == "" ]]; then + echoerr "Error: you need to provide a host and port to test." + usage +fi + +WAITFORIT_TIMEOUT=${WAITFORIT_TIMEOUT:-15} +WAITFORIT_STRICT=${WAITFORIT_STRICT:-0} +WAITFORIT_CHILD=${WAITFORIT_CHILD:-0} +WAITFORIT_QUIET=${WAITFORIT_QUIET:-0} + +# Check to see if timeout is from busybox? +WAITFORIT_TIMEOUT_PATH=$(type -p timeout) +WAITFORIT_TIMEOUT_PATH=$(realpath $WAITFORIT_TIMEOUT_PATH 2>/dev/null || readlink -f $WAITFORIT_TIMEOUT_PATH) + +WAITFORIT_BUSYTIMEFLAG="" +if [[ $WAITFORIT_TIMEOUT_PATH =~ "busybox" ]]; then + WAITFORIT_ISBUSY=1 + # Check if busybox timeout uses -t flag + # (recent Alpine versions don't support -t anymore) + if timeout &>/dev/stdout | grep -q -e '-t '; then + WAITFORIT_BUSYTIMEFLAG="-t" + fi +else + WAITFORIT_ISBUSY=0 +fi + +if [[ $WAITFORIT_CHILD -gt 0 ]]; then + wait_for + WAITFORIT_RESULT=$? + exit $WAITFORIT_RESULT +else + if [[ $WAITFORIT_TIMEOUT -gt 0 ]]; then + wait_for_wrapper + WAITFORIT_RESULT=$? + else + wait_for + WAITFORIT_RESULT=$? + fi +fi + +if [[ $WAITFORIT_CLI != "" ]]; then + if [[ $WAITFORIT_RESULT -ne 0 && $WAITFORIT_STRICT -eq 1 ]]; then + echoerr "$WAITFORIT_cmdname: strict mode, refusing to execute subprocess" + exit $WAITFORIT_RESULT + fi + exec "${WAITFORIT_CLI[@]}" +else + exit $WAITFORIT_RESULT +fi diff --git a/tools/wait_for_url.py b/tools/wait_for_url.py new file mode 100755 index 000000000..751bdc00b --- /dev/null +++ b/tools/wait_for_url.py @@ -0,0 +1,40 @@ +#!/usr/bin/env python3 +# Use this script to wait for a given URL to become available +import argparse +import subprocess +import time +import sys + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("url") + parser.add_argument("--interval", type=int, default=3) + parser.add_argument("--max-attempts", type=int, default=5) + args = parser.parse_args() + + url = args.url + interval = args.interval + max_attempts = args.max_attempts + + start_time = time.time() + + print(f"wait-for-url.py: waiting {interval * max_attempts} seconds for {url}") + + for _ in range(max_attempts): + result = subprocess.run( + ["curl", "--output", "/dev/null", "--silent", "--fail", url] + ) + + if result.returncode == 0: + elapsed = time.time() - start_time + print(f"wait-for-url.py: {url} is available after {elapsed:.0f} seconds") + break + + print(".", end="", flush=True) + time.sleep(interval) + else: + print() + print(f"wait-for-it.py: {url} failed to become available") + sys.exit(1) + + sys.exit(0)