Rework

.github/workflows/metabase_export.yml

@@ -28,8 +28,8 @@ jobs:
 
       - name: Init CI environment variables
         run: |
-          echo "DATABASE_URL=${{ secrets.METABASE_EXPORT_SRC_DATABASE_URL }}" >> .env
-          echo "METABASE_DATABASE_URL=${{ secrets.METABASE_EXPORT_DEST_DATABASE_URL }}" >> .env
+          echo "DATABASE_URL=${{ secrets.METABASE_EXPORT_DATABASE_URL }}" >> .env
+          echo "METABASE_DATABASE_URL=${{ secrets.METABASE_EXPORT_METABASE_DATABASE_URL }}" >> .env
 
       - name: Run export
         run: make ci_metabase_export

.github/workflows/metabase_migrate.yml

@@ -0,0 +1,40 @@
+name: Metabase Migrate
+
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - main
+    paths:
+      - 'src/Infrastructure/Persistence/Doctrine/MetabaseMigrations/**'
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v1
+
+      - name: Setup PHP with PECL extension
+        uses: shivammathur/setup-php@v2
+        with:
+          php-version: '8.2'
+
+      - name: Get Composer Cache Directory
+        id: composer-cache
+        run: |
+          echo "dir=$(composer config cache-files-dir)" >> $GITHUB_OUTPUT
+
+      - uses: actions/cache@v3
+        with:
+          path: ${{ steps.composer-cache.outputs.dir }}
+          key: ${{ runner.os }}-composer-${{ hashFiles('**/composer.lock') }}
+          restore-keys: |
+            ${{ runner.os }}-composer-
+
+      - name: Init environment variables
+        run: |
+          echo "METABASE_DATABASE_URL=${{ secrets.METABASE_MIGRATIONS_METABASE_DATABASE_URL }}" >> .env
+
+      - name: CI
+        run: make ci_metabase_migrate BIN_COMPOSER="composer" BIN_CONSOLE="php bin/console"

Makefile

@@ -67,6 +67,7 @@ dbinstall: ## Setup databases
 	make data_install
 	make console CMD="doctrine:database:create --env=test --if-not-exists"
 	make dbmigrate ARGS="--env=test"
+	make metabase_migrate ARGS="--env=test"
 	make data_install ARGS="--env=test"
 	make dbfixtures
 
@@ -107,6 +108,12 @@ bdtopo_migrate_redo: ## Revert db migrations for bdtopo and run them again
 	# Re-run migrations from there
 	make bdtopo_migrate
 
+metabase_migration: ## Generate new migration for metabase
+	${BIN_CONSOLE} doctrine:migrations:generate --configuration ./config/packages/metabase/doctrine_migrations.yaml
+
+metabase_migrate: ## Run db migrations for metabase
+	${BIN_CONSOLE} doctrine:migrations:migrate -n --all-or-nothing --configuration ./config/packages/metabase/doctrine_migrations.yaml ${ARGS}
+
 dbshell: ## Connect to the database
 	docker compose exec database psql postgresql://dialog:dialog@database:5432/dialog
 
@@ -280,6 +287,10 @@ ci_bdtopo_migrate: ## Run CI steps for BD TOPO Migrate workflow
 	make composer CMD="install -n --prefer-dist"
 	make bdtopo_migrate
 
+ci_metabase_migrate: ## Run CI steps for Metabase Migrate workflow
+	make composer CMD="install -n --prefer-dist"
+	make metabase_migrate
+
 ci_metabase_export: ## Export data to Metabase
 	scalingo login --ssh --ssh-identity ~/.ssh/id_rsa
 	./tools/scalingodbtunnel dialog-metabase --host-url --port 10001 & ./tools/wait-for-it.sh 127.0.0.1:10001

config/packages/doctrine.yaml

@@ -36,6 +36,8 @@ doctrine:
                         alias: 'App\Domain'
             bdtopo:
                 connection: bdtopo
+            metabase:
+                connection: metabase
 
 when@test:
     doctrine:

config/packages/metabase/doctrine_migrations.yaml

@@ -0,0 +1,3 @@
+migrations_paths:
+    App\Infrastructure\Persistence\Doctrine\MetabaseMigrations: 'src/Infrastructure/Persistence/Doctrine/MetabaseMigrations'
+em: metabase

docs/tools/metabase.md

@@ -18,12 +18,25 @@ La collecte des données d'indicateurs est réalisée au moyen d'une la commande
 
 L'export Metabase peut être déclenché via [GitHub Actions](./github_actions.md) à l'aide du workflow [`metabase_export.yml`](../../.github/workflows/metabase_export.yml).
 
+## Tester l'export en local
+
+Vous pouvez tester l'export en local en configurant votre `.env.local` comme ceci :
+
+```bash
+METABASE_DATABASE_URL="postgresql://dialog:dialog@database:5432/dialog"
+```
+
+Lancez ensuite `make console CMD="app:metabase:export"`. Cela aura pour effet de calculer et charger les indicateurs directement dans votre base locale `dialog`.
+
+La visualisation des graphiques Metabase à partir de ces données n'est pas possible, mais vous pourrez au moins explorer les données brutes dans les tables commençant par `analytics_`.
+
 ### Configuration de la GitHub Action
 
 La configuration de la GitHub Action passe par diverses variables d'environnement listées ci-dessous :
 
 | Variable d'environnement | Configuration | Description |
 |---|---|---|
-| `METABASE_EXPORT_SRC_DATABASE_URL` | [Secret](https://docs.github.com/fr/actions/security-guides/using-secrets-in-github-actions) au sens GitHub Actions | L'URL d'accès à la base de données applicative par la CI (`./tools/scalingodbtunnel dialog --host-url --port 10001`) |
-| `METABASE_EXPORT_DEST_DATABASE_URL` | Secret | L'URL d'accès à la base de données Metabase par la CI (`./tools/scalingodbtunnel dialog-metabase --host-url --port 10001`) |
+| `METABASE_MIGRATIONS_METABASE_DATABASE_URL` | [Secret](https://docs.github.com/fr/actions/security-guides/using-secrets-in-github-actions) au sens GitHub Actions | L'URL d'accès à la base de données Metabase par la CI, afin d'exécuter les migrations (`./tools/scalingodbtunnel dialog-metabase --host-url --port 10001`) |
+| `METABASE_EXPORT_DATABASE_URL` | [Secret](https://docs.github.com/fr/actions/security-guides/using-secrets-in-github-actions) au sens GitHub Actions | L'URL d'accès à la base de données applicative par la CI (`./tools/scalingodbtunnel dialog --host-url --port 10001`) |
+| `METABASE_EXPORT_METABASE_DATABASE_URL` | Secret | L'URL d'accès à la base de données Metabase par la CI (`./tools/scalingodbtunnel dialog-metabase --host-url --port 10001`) |
 | `GH_SCALINGO_SSH_PRIVATE_KEY` | Secret | Clé SSH privée permettant l'accès à Scalingo par la CI |

phpunit.xml.dist

@@ -25,6 +25,7 @@
             <directory suffix=".php">src/Infrastructure/Persistence/Doctrine/Mapping</directory>
             <directory suffix=".php">src/Infrastructure/Persistence/Doctrine/Migrations</directory>
             <directory suffix=".php">src/Infrastructure/Persistence/Doctrine/BdTopoMigrations</directory>
+            <directory suffix=".php">src/Infrastructure/Persistence/Doctrine/MetabaseMigrations</directory>
             <directory suffix=".php">src/Infrastructure/Persistence/Doctrine/PostGIS/Event</directory>
             <file>src/Infrastructure/Adapter/CommandBus.php</file>
             <file>src/Infrastructure/Adapter/IdFactory.php</file>

src/Domain/Statistics/Repository/StatisticsRepositoryInterface.php

@@ -0,0 +1,10 @@
+<?php
+
+declare(strict_types=1);
+
+namespace App\Domain\Statistics\Repository;
+
+interface StatisticsRepositoryInterface
+{
+    public function addUserActiveStatistics(\DateTimeImmutable $now): void;
+}

src/Domain/User/Repository/UserRepositoryInterface.php

@@ -15,4 +15,6 @@ public function countUsers(): int;
     public function add(User $user): User;
 
     public function remove(User $user): void;
+
+    public function findAllForStatistics(): array;
 }

src/Infrastructure/Persistence/Doctrine/MetabaseMigrations/Version20241216132451.php

@@ -0,0 +1,38 @@
+<?php
+
+declare(strict_types=1);
+
+namespace App\Infrastructure\Persistence\Doctrine\MetabaseMigrations;
+
+use Doctrine\DBAL\Schema\Schema;
+use Doctrine\Migrations\AbstractMigration;
+
+final class Version20241216132451 extends AbstractMigration
+{
+    public function getDescription(): string
+    {
+        return '';
+    }
+
+    public function up(Schema $schema): void
+    {
+        $this->addSql(
+            'CREATE TABLE IF NOT EXISTS analytics_user_active (
+                id UUID NOT NULL,
+                uploaded_at TIMESTAMP(0),
+                last_active_at TIMESTAMP(0),
+                PRIMARY KEY(id)
+            );',
+        );
+
+        $this->addSql(
+            'CREATE INDEX IF NOT EXISTS idx_analytics_user_active_uploaded_at
+            ON analytics_user_active (uploaded_at);',
+        );
+    }
+
+    public function down(Schema $schema): void
+    {
+        $this->addSql('DROP TABLE IF EXISTS analytics_user_active');
+    }
+}

src/Infrastructure/Persistence/Doctrine/Repository/Statistics/StatisticsRepository.php

@@ -0,0 +1,42 @@
+<?php
+
+declare(strict_types=1);
+
+namespace App\Infrastructure\Persistence\Doctrine\Repository\Statistics;
+
+use App\Domain\Statistics\Repository\StatisticsRepositoryInterface;
+use App\Domain\User\Repository\UserRepositoryInterface;
+use Doctrine\DBAL\Connection;
+
+final class StatisticsRepository implements StatisticsRepositoryInterface
+{
+    public function __construct(
+        private UserRepositoryInterface $userRepository,
+        private Connection $metabaseConnection,
+    ) {
+    }
+
+    public function addUserActiveStatistics(\DateTimeInterface $now): void
+    {
+        // À chaque export des statistiques, on ajoute la liste des dates de dernière activité pour chaque utilisateur, et la date d'exécution.
+        // Dans Metabase cela permet de calculer le nombre d'utilisateurs actifs au moment de chaque exécution.
+        // (Par exemple avec un filtre : "[last_active_at] >= [uploaded_at] - 7 jours", puis en groupant sur le uploaded_at.)
+        $userRows = $this->userRepository->findAllForStatistics();
+        $this->bulkInsertUserActiveStatistics($now, $userRows);
+    }
+
+    private function bulkInsertUserActiveStatistics(\DateTimeInterface $now, array $userRows): void
+    {
+        $stmt = $this->metabaseConnection->prepare(
+            'INSERT INTO analytics_user_active(id, uploaded_at, last_active_at)
+            VALUES (:id, (:uploadedAt)::timestamp(0), (:lastActiveAt)::timestamp(0))',
+        );
+
+        foreach ($userRows as $row) {
+            $stmt->bindValue('id', $row['uuid']);
+            $stmt->bindValue('uploadedAt', $now->format(\DateTimeInterface::ATOM));
+            $stmt->bindValue('lastActiveAt', $row['lastActiveAt']?->format(\DateTimeInterface::ATOM));
+            $stmt->execute();
+        }
+    }
+}

src/Infrastructure/Persistence/Doctrine/Repository/User/UserRepository.php

@@ -48,4 +48,12 @@ public function countUsers(): int
             ->getQuery()
             ->getSingleScalarResult();
     }
+
+    public function findAllForStatistics(): array
+    {
+        return $this->createQueryBuilder('u')
+            ->select('u.uuid, u.lastActiveAt')
+            ->getQuery()
+            ->getResult();
+    }
 }

src/Infrastructure/Symfony/Command/RunMetabaseExportCommand.php

@@ -5,23 +5,22 @@
 namespace App\Infrastructure\Symfony\Command;
 
 use App\Application\DateUtilsInterface;
-use Doctrine\DBAL\Connection;
+use App\Domain\Statistics\Repository\StatisticsRepositoryInterface;
 use Symfony\Component\Console\Attribute\AsCommand;
 use Symfony\Component\Console\Command\Command;
 use Symfony\Component\Console\Input\InputInterface;
 use Symfony\Component\Console\Output\OutputInterface;
 
 #[AsCommand(
     name: 'app:metabase:export',
-    description: 'Export indicators to Metabase',
+    description: 'Export statistics to Metabase',
     hidden: false,
 )]
 class RunMetabaseExportCommand extends Command
 {
     public function __construct(
         private DateUtilsInterface $dateUtils,
-        private Connection $connection,
-        private Connection $metabaseConnection,
+        private StatisticsRepositoryInterface $statisticsRepository,
     ) {
         parent::__construct();
     }
@@ -30,34 +29,8 @@ public function execute(InputInterface $input, OutputInterface $output): int
     {
         $now = $this->dateUtils->getNow();
 
-        $this->exportActiveUsers($now);
+        $this->statisticsRepository->addUserActiveStatistics($now);
 
         return Command::SUCCESS;
     }
-
-    private function exportActiveUsers(\DateTimeInterface $now): void
-    {
-        // À chaque exécution, on ajoute la liste des dates de dernière activité pour chaque utilisateur, et la date d'exécution.
-        // Dans Metabase cela permet de calculer le nombre d'utilisateurs actif au moment de chaque exécution.
-        // (Par exemple avec un filtre : "[last_active_at] >= [uploaded_at] - 7 jours", puis en groupant sur le uploaded_at.)
-        $this->metabaseConnection->executeQuery(
-            'CREATE TABLE IF NOT EXISTS analytics_user_active (id UUID NOT NULL, uploaded_at TIMESTAMP(0), last_active_at TIMESTAMP(0), PRIMARY KEY(id));',
-        );
-        $this->metabaseConnection->executeQuery(
-            'CREATE INDEX IF NOT EXISTS idx_analytics_user_active_uploaded_at ON analytics_user_active (uploaded_at);',
-        );
-
-        $userRows = $this->connection->fetchAllAssociative(
-            'SELECT uuid_generate_v4() AS id, last_active_at FROM "user"',
-        );
-
-        $stmt = $this->metabaseConnection->prepare('INSERT INTO analytics_user_active(id, uploaded_at, last_active_at) VALUES (:id, (:uploaded_at)::timestamp(0), :last_active_at)');
-
-        foreach ($userRows as $row) {
-            $stmt->bindValue('id', $row['id']);
-            $stmt->bindValue('uploaded_at', $now->format(\DateTimeInterface::ATOM));
-            $stmt->bindValue('last_active_at', $row['last_active_at']);
-            $stmt->execute();
-        }
-    }
 }