Exécute l'export Metabase depuis PHP

.env

@@ -22,6 +22,7 @@ APP_EUDONET_PARIS_BASE_URL=https://eudonet-partage.apps.paris.fr
 APP_BAC_IDF_DECREES_FILE=data/bac_idf/decrees.json
 APP_BAC_IDF_CITIES_FILE=data/bac_idf/cities.csv
 DATABASE_URL="postgresql://dialog:dialog@database:5432/dialog"
+METABASE_DATABASE_URL="postgresql://dialog:dialog@database:5432/dialog"
 REDIS_URL="redis://redis:6379"
 API_ADRESSE_BASE_URL=https://api-adresse.data.gouv.fr
 APP_IGN_GEOCODER_BASE_URL=https://data.geopf.fr

.github/workflows/ci.yml

@@ -61,6 +61,7 @@ jobs:
         run: |
           echo "DATABASE_URL=postgresql://dialog:dialog@localhost:5432/dialog" >> .env
           echo "BDTOPO_DATABASE_URL=${{ secrets.BDTOPO_DATABASE_URL }}" >> .env
+          echo "METABASE_DATABASE_URL=postgresql://dialog:dialog@localhost:5432/dialog" >> .env
           echo "REDIS_URL=redis://localhost:6379" >> .env
           echo "APP_STORAGE_SOURCE=memory.storage" >> .env
 

.github/workflows/metabase_export.yml

@@ -26,10 +26,10 @@ jobs:
         run: |
           ssh-keyscan -H ssh.osc-fr1.scalingo.com >> ~/.ssh/known_hosts
 
+      - name: Init CI environment variables
+        run: |
+          echo "DATABASE_URL=${{ secrets.METABASE_EXPORT_DATABASE_URL }}" >> .env
+          echo "METABASE_DATABASE_URL=${{ secrets.METABASE_EXPORT_METABASE_DATABASE_URL }}" >> .env
+
       - name: Run export
         run: make ci_metabase_export
-        env:
-          METABASE_SRC_APP: ${{ vars.METABASE_EXPORT_SRC_APP }}
-          METABASE_SRC_DATABASE_URL: ${{ secrets.METABASE_EXPORT_SRC_DATABASE_URL }}
-          METABASE_DEST_APP: ${{ vars.METABASE_EXPORT_DEST_APP }}
-          METABASE_DEST_DATABASE_URL: ${{ secrets.METABASE_EXPORT_DEST_DATABASE_URL }}

.github/workflows/metabase_migrate.yml

@@ -0,0 +1,40 @@
+name: Metabase Migrate
+
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - main
+    paths:
+      - 'src/Infrastructure/Persistence/Doctrine/MetabaseMigrations/**'
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v1
+
+      - name: Setup PHP with PECL extension
+        uses: shivammathur/setup-php@v2
+        with:
+          php-version: '8.2'
+
+      - name: Get Composer Cache Directory
+        id: composer-cache
+        run: |
+          echo "dir=$(composer config cache-files-dir)" >> $GITHUB_OUTPUT
+
+      - uses: actions/cache@v3
+        with:
+          path: ${{ steps.composer-cache.outputs.dir }}
+          key: ${{ runner.os }}-composer-${{ hashFiles('**/composer.lock') }}
+          restore-keys: |
+            ${{ runner.os }}-composer-
+
+      - name: Init environment variables
+        run: |
+          echo "METABASE_DATABASE_URL=${{ secrets.METABASE_MIGRATIONS_METABASE_DATABASE_URL }}" >> .env
+
+      - name: CI
+        run: make ci_metabase_migrate BIN_COMPOSER="composer" BIN_CONSOLE="php bin/console"

Makefile

@@ -67,6 +67,7 @@ dbinstall: ## Setup databases
 	make data_install
 	make console CMD="doctrine:database:create --env=test --if-not-exists"
 	make dbmigrate ARGS="--env=test"
+	make metabase_migrate ARGS="--env=test"
 	make data_install ARGS="--env=test"
 	make dbfixtures
 
@@ -107,6 +108,12 @@ bdtopo_migrate_redo: ## Revert db migrations for bdtopo and run them again
 	# Re-run migrations from there
 	make bdtopo_migrate
 
+metabase_migration: ## Generate new migration for metabase
+	${BIN_CONSOLE} doctrine:migrations:generate --configuration ./config/packages/metabase/doctrine_migrations.yaml
+
+metabase_migrate: ## Run db migrations for metabase
+	${BIN_CONSOLE} doctrine:migrations:migrate -n --all-or-nothing --configuration ./config/packages/metabase/doctrine_migrations.yaml ${ARGS}
+
 dbshell: ## Connect to the database
 	docker compose exec database psql postgresql://dialog:dialog@database:5432/dialog
 
@@ -280,10 +287,14 @@ ci_bdtopo_migrate: ## Run CI steps for BD TOPO Migrate workflow
 	make composer CMD="install -n --prefer-dist"
 	make bdtopo_migrate
 
+ci_metabase_migrate: ## Run CI steps for Metabase Migrate workflow
+	make composer CMD="install -n --prefer-dist"
+	make metabase_migrate
+
 ci_metabase_export: ## Export data to Metabase
 	scalingo login --ssh --ssh-identity ~/.ssh/id_rsa
-	./tools/scalingodbtunnel ${METABASE_DEST_APP} --host-url --port 10001 & ./tools/wait-for-it.sh 127.0.0.1:10001
-	./tools/metabase-export.sh ${METABASE_SRC_DATABASE_URL} ${METABASE_DEST_DATABASE_URL}
+	./tools/scalingodbtunnel dialog-metabase --host-url --port 10001 & ./tools/wait-for-it.sh 127.0.0.1:10001
+	make console CMD="app:metabase:export"
 
 ##
 ## ----------------

config/packages/doctrine.yaml

@@ -10,6 +10,9 @@ doctrine:
             bdtopo:
                 url: '%env(BDTOPO_DATABASE_URL)%'
                 use_savepoints: true
+            metabase:
+                url: '%env(METABASE_DATABASE_URL)%'
+                use_savepoints: true
     orm:
         auto_generate_proxy_classes: true
         default_entity_manager: default
@@ -33,12 +36,18 @@ doctrine:
                         alias: 'App\Domain'
             bdtopo:
                 connection: bdtopo
+            metabase:
+                connection: metabase
 
 when@test:
     doctrine:
         dbal:
-            # "TEST_TOKEN" is typically set by ParaTest
-            dbname_suffix: '_test%env(default::TEST_TOKEN)%'
+            connections:
+                default:
+                    # "TEST_TOKEN" is typically set by ParaTest
+                    dbname_suffix: '_test%env(default::TEST_TOKEN)%'
+                metabase:
+                    dbname_suffix: '_test%env(default::TEST_TOKEN)%'
 
 when@prod:
     doctrine:

config/packages/metabase/doctrine_migrations.yaml

@@ -0,0 +1,3 @@
+migrations_paths:
+    App\Infrastructure\Persistence\Doctrine\MetabaseMigrations: 'src/Infrastructure/Persistence/Doctrine/MetabaseMigrations'
+em: metabase

docs/tools/metabase.md

@@ -12,20 +12,31 @@ Le Metabase de DiaLog est hébergé sur Scalingo sous l'application `dialog-meta
 
 Cette application dispose de sa propre base de données où nous stockons les données nécessaires au calcul des indicateurs, conformément aux [recommendations Beta](https://doc.incubateur.net/communaute/les-outils-de-la-communaute/autres-services/metabase/metabase#connecter-metabase-a-une-base-de-donnees-anonymisee)
 
-La collecte des données d'indicateurs est réalisée au moyen d'un [script](../../tools/metabase-export.sh). Ce script exécute des requêtes SQL depuis la base Metabase vers la base applicative. Pour cela un utilisateur `dialog_metabase` avec droits en lecture seule a été créé sur la base applicative (identifiants dans le Vaultwarden de l'équipe DiaLog).
+La collecte des données d'indicateurs est réalisée au moyen d'une la commande Symfony `app:metabase:export`. Cette commande rassemble les données sources (requêtes à la base de données, requêtes HTTP, ou autres opérations...) puis les upload vers la base de données PostgreSQL de l'instance Metabase. 
 
 ## Lancer l'export depuis GitHub Actions
 
 L'export Metabase peut être déclenché via [GitHub Actions](./github_actions.md) à l'aide du workflow [`metabase_export.yml`](../../.github/workflows/metabase_export.yml).
 
+## Tester l'export en local
+
+Vous pouvez tester l'export en local en configurant votre `.env.local` comme ceci :
+
+```bash
+METABASE_DATABASE_URL="postgresql://dialog:dialog@database:5432/dialog"
+```
+
+Lancez ensuite `make console CMD="app:metabase:export"`. Cela aura pour effet de calculer et charger les indicateurs directement dans votre base locale `dialog`.
+
+La visualisation des graphiques Metabase à partir de ces données n'est pas possible, mais vous pourrez au moins explorer les données brutes dans les tables commençant par `analytics_`.
+
 ### Configuration de la GitHub Action
 
 La configuration de la GitHub Action passe par diverses variables d'environnement listées ci-dessous :
 
 | Variable d'environnement | Configuration | Description |
 |---|---|---|
-| `METABASE_EXPORT_SRC_APP` | [Variable](https://docs.github.com/fr/actions/learn-github-actions/variables) au sens GitHub Actions | `dialog` (pour la production) |
-| `METABASE_EXPORT_SRC_DATABASE_URL` | [Secret](https://docs.github.com/fr/actions/security-guides/using-secrets-in-github-actions) au sens GitHub Actions | L'URL d'accès à la base de données applicative par la DB Metabase : utiliser la `METABASE_EXPORT_SRC_DATABASE_URL` de l'app `dialog` |
-| `METABASE_EXPORT_DEST_APP` | Variable | `dialog-metabase` |
-| `METABASE_EXPORT_DEST_DATABASE_URL` | Secret | L'URL d'accès à la base de données Metabase par la CI (`./tools/scalingodbtunnel dialog-metabase  --host-url --port 10001`) |
+| `METABASE_MIGRATIONS_METABASE_DATABASE_URL` | [Secret](https://docs.github.com/fr/actions/security-guides/using-secrets-in-github-actions) au sens GitHub Actions | L'URL d'accès à la base de données Metabase par la CI, afin d'exécuter les migrations (`./tools/scalingodbtunnel dialog-metabase --host-url --port 10001`) |
+| `METABASE_EXPORT_DATABASE_URL` | [Secret](https://docs.github.com/fr/actions/security-guides/using-secrets-in-github-actions) au sens GitHub Actions | L'URL d'accès à la base de données applicative par la CI (`./tools/scalingodbtunnel dialog --host-url --port 10001`) |
+| `METABASE_EXPORT_METABASE_DATABASE_URL` | Secret | L'URL d'accès à la base de données Metabase par la CI (`./tools/scalingodbtunnel dialog-metabase --host-url --port 10001`) |
 | `GH_SCALINGO_SSH_PRIVATE_KEY` | Secret | Clé SSH privée permettant l'accès à Scalingo par la CI |

phpunit.xml.dist

@@ -25,6 +25,7 @@
             <directory suffix=".php">src/Infrastructure/Persistence/Doctrine/Mapping</directory>
             <directory suffix=".php">src/Infrastructure/Persistence/Doctrine/Migrations</directory>
             <directory suffix=".php">src/Infrastructure/Persistence/Doctrine/BdTopoMigrations</directory>
+            <directory suffix=".php">src/Infrastructure/Persistence/Doctrine/MetabaseMigrations</directory>
             <directory suffix=".php">src/Infrastructure/Persistence/Doctrine/PostGIS/Event</directory>
             <file>src/Infrastructure/Adapter/CommandBus.php</file>
             <file>src/Infrastructure/Adapter/IdFactory.php</file>

src/Domain/Statistics/Repository/StatisticsRepositoryInterface.php

@@ -0,0 +1,10 @@
+<?php
+
+declare(strict_types=1);
+
+namespace App\Domain\Statistics\Repository;
+
+interface StatisticsRepositoryInterface
+{
+    public function addUserActiveStatistics(\DateTimeImmutable $now): void;
+}

src/Domain/User/Repository/UserRepositoryInterface.php

@@ -15,4 +15,6 @@ public function countUsers(): int;
     public function add(User $user): User;
 
     public function remove(User $user): void;
+
+    public function findAllForStatistics(): array;
 }

src/Domain/User/User.php

@@ -88,9 +88,11 @@ public function getLastActiveAt(): ?\DateTimeInterface
         return $this->lastActiveAt;
     }
 
-    public function setLastActiveAt(\DateTimeInterface $date): void
+    public function setLastActiveAt(\DateTimeInterface $date): self
     {
         $this->lastActiveAt = $date;
+
+        return $this;
     }
 
     public function __toString(): string

src/Infrastructure/Persistence/Doctrine/Fixtures/UserFixture.php

@@ -23,14 +23,16 @@ public function load(ObjectManager $manager): void
             ->setEmail(self::MAIN_ORG_USER_EMAIL)
             ->setPassword(self::PASSWORD)
             ->setRoles([UserRolesEnum::ROLE_USER->value])
-            ->setRegistrationDate(new \DateTimeImmutable('2024-03-01'));
+            ->setRegistrationDate(new \DateTimeImmutable('2024-03-01'))
+            ->setLastActiveAt(new \DateTimeImmutable('2024-06-07'));
 
         $mainOtherAdmin = (new User('5bc831a3-7a09-44e9-aefa-5ce3588dac33'))
             ->setFullName('Mathieu FERNANDEZ')
             ->setEmail(self::MAIN_ORG_ADMIN_EMAIL)
             ->setPassword(self::PASSWORD)
             ->setRoles([UserRolesEnum::ROLE_SUPER_ADMIN->value])
-            ->setRegistrationDate(new \DateTimeImmutable('2024-04-02'));
+            ->setRegistrationDate(new \DateTimeImmutable('2024-04-02'))
+            ->setLastActiveAt(new \DateTimeImmutable('2024-06-08'));
 
         $otherOrgUser = (new User('d47badd9-989e-472b-a80e-9df642e93880'))
             ->setFullName('Florimond MANCA')

src/Infrastructure/Persistence/Doctrine/MetabaseMigrations/Version20241216132451.php

@@ -0,0 +1,38 @@
+<?php
+
+declare(strict_types=1);
+
+namespace App\Infrastructure\Persistence\Doctrine\MetabaseMigrations;
+
+use Doctrine\DBAL\Schema\Schema;
+use Doctrine\Migrations\AbstractMigration;
+
+final class Version20241216132451 extends AbstractMigration
+{
+    public function getDescription(): string
+    {
+        return '';
+    }
+
+    public function up(Schema $schema): void
+    {
+        $this->addSql(
+            'CREATE TABLE IF NOT EXISTS analytics_user_active (
+                id UUID NOT NULL,
+                uploaded_at TIMESTAMP(0),
+                last_active_at TIMESTAMP(0),
+                PRIMARY KEY(id)
+            );',
+        );
+
+        $this->addSql(
+            'CREATE INDEX IF NOT EXISTS idx_analytics_user_active_uploaded_at
+            ON analytics_user_active (uploaded_at);',
+        );
+    }
+
+    public function down(Schema $schema): void
+    {
+        $this->addSql('DROP TABLE IF EXISTS analytics_user_active');
+    }
+}

src/Infrastructure/Persistence/Doctrine/Repository/Statistics/StatisticsRepository.php

@@ -0,0 +1,42 @@
+<?php
+
+declare(strict_types=1);
+
+namespace App\Infrastructure\Persistence\Doctrine\Repository\Statistics;
+
+use App\Domain\Statistics\Repository\StatisticsRepositoryInterface;
+use App\Domain\User\Repository\UserRepositoryInterface;
+use Doctrine\DBAL\Connection;
+
+final class StatisticsRepository implements StatisticsRepositoryInterface
+{
+    public function __construct(
+        private UserRepositoryInterface $userRepository,
+        private Connection $metabaseConnection,
+    ) {
+    }
+
+    public function addUserActiveStatistics(\DateTimeInterface $now): void
+    {
+        // À chaque export des statistiques, on ajoute la liste des dates de dernière activité pour chaque utilisateur, et la date d'exécution.
+        // Dans Metabase cela permet de calculer le nombre d'utilisateurs actifs au moment de chaque exécution.
+        // (Par exemple avec un filtre : "[last_active_at] >= [uploaded_at] - 7 jours", puis en groupant sur le uploaded_at.)
+        $userRows = $this->userRepository->findAllForStatistics();
+        $this->bulkInsertUserActiveStatistics($now, $userRows);
+    }
+
+    private function bulkInsertUserActiveStatistics(\DateTimeInterface $now, array $userRows): void
+    {
+        $stmt = $this->metabaseConnection->prepare(
+            'INSERT INTO analytics_user_active(id, uploaded_at, last_active_at)
+            VALUES (:id, (:uploadedAt)::timestamp(0), (:lastActiveAt)::timestamp(0))',
+        );
+
+        foreach ($userRows as $row) {
+            $stmt->bindValue('id', $row['uuid']);
+            $stmt->bindValue('uploadedAt', $now->format(\DateTimeInterface::ATOM));
+            $stmt->bindValue('lastActiveAt', $row['lastActiveAt']?->format(\DateTimeInterface::ATOM));
+            $stmt->execute();
+        }
+    }
+}

src/Infrastructure/Persistence/Doctrine/Repository/User/UserRepository.php

@@ -48,4 +48,12 @@ public function countUsers(): int
             ->getQuery()
             ->getSingleScalarResult();
     }
+
+    public function findAllForStatistics(): array
+    {
+        return $this->createQueryBuilder('u')
+            ->select('u.uuid, u.lastActiveAt')
+            ->getQuery()
+            ->getResult();
+    }
 }

src/Infrastructure/Symfony/Command/RunMetabaseExportCommand.php

@@ -0,0 +1,36 @@
+<?php
+
+declare(strict_types=1);
+
+namespace App\Infrastructure\Symfony\Command;
+
+use App\Application\DateUtilsInterface;
+use App\Domain\Statistics\Repository\StatisticsRepositoryInterface;
+use Symfony\Component\Console\Attribute\AsCommand;
+use Symfony\Component\Console\Command\Command;
+use Symfony\Component\Console\Input\InputInterface;
+use Symfony\Component\Console\Output\OutputInterface;
+
+#[AsCommand(
+    name: 'app:metabase:export',
+    description: 'Export statistics to Metabase',
+    hidden: false,
+)]
+class RunMetabaseExportCommand extends Command
+{
+    public function __construct(
+        private DateUtilsInterface $dateUtils,
+        private StatisticsRepositoryInterface $statisticsRepository,
+    ) {
+        parent::__construct();
+    }
+
+    public function execute(InputInterface $input, OutputInterface $output): int
+    {
+        $now = $this->dateUtils->getNow();
+
+        $this->statisticsRepository->addUserActiveStatistics($now);
+
+        return Command::SUCCESS;
+    }
+}

tests/Integration/Infrastructure/Controller/Regulation/AddRegulationControllerTest.php

@@ -38,7 +38,7 @@ public function testAdd(): void
 
         /** @var UserRepositoryInterface */
         $userRepository = static::getContainer()->get(UserRepositoryInterface::class);
-        $this->assertNull($userRepository->findOneByEmail($email)->getLastActiveAt());
+        $this->assertEquals(new \DateTimeImmutable('2024-06-07'), $userRepository->findOneByEmail($email)->getLastActiveAt());
 
         // Get the raw values.
         $values = $form->getPhpValues();
@@ -56,6 +56,7 @@ public function testAdd(): void
         $crawler = $client->request($form->getMethod(), $form->getUri(), $values, $form->getPhpFiles());
 
         $this->assertResponseStatusCodeSame(303);
+        // Filled with DateUtilsMock::getNow()
         $this->assertEquals(new \DateTimeImmutable('2023-06-09'), $userRepository->findOneByEmail($email)->getLastActiveAt());
         $client->followRedirect();