diff --git a/.github/workflows/metabase_export.yml b/.github/workflows/metabase_export.yml
index 19d42e986..a17681b47 100644
--- a/.github/workflows/metabase_export.yml
+++ b/.github/workflows/metabase_export.yml
@@ -28,8 +28,8 @@ jobs:
- name: Init CI environment variables
run: |
- echo "DATABASE_URL=${{ secrets.METABASE_EXPORT_SRC_DATABASE_URL }}" >> .env
- echo "METABASE_DATABASE_URL=${{ secrets.METABASE_EXPORT_DEST_DATABASE_URL }}" >> .env
+ echo "DATABASE_URL=${{ secrets.METABASE_EXPORT_DATABASE_URL }}" >> .env
+ echo "METABASE_DATABASE_URL=${{ secrets.METABASE_EXPORT_METABASE_DATABASE_URL }}" >> .env
- name: Run export
run: make ci_metabase_export
diff --git a/.github/workflows/metabase_migrate.yml b/.github/workflows/metabase_migrate.yml
new file mode 100644
index 000000000..68edc0672
--- /dev/null
+++ b/.github/workflows/metabase_migrate.yml
@@ -0,0 +1,40 @@
+name: Metabase Migrate
+
+on:
+ workflow_dispatch:
+ push:
+ branches:
+ - main
+ paths:
+ - 'src/Infrastructure/Persistence/Doctrine/MetabaseMigrations/**'
+
+jobs:
+ build:
+ runs-on: ubuntu-latest
+
+ steps:
+ - uses: actions/checkout@v1
+
+ - name: Setup PHP with PECL extension
+ uses: shivammathur/setup-php@v2
+ with:
+ php-version: '8.2'
+
+ - name: Get Composer Cache Directory
+ id: composer-cache
+ run: |
+ echo "dir=$(composer config cache-files-dir)" >> $GITHUB_OUTPUT
+
+ - uses: actions/cache@v3
+ with:
+ path: ${{ steps.composer-cache.outputs.dir }}
+ key: ${{ runner.os }}-composer-${{ hashFiles('**/composer.lock') }}
+ restore-keys: |
+ ${{ runner.os }}-composer-
+
+ - name: Init environment variables
+ run: |
+ echo "METABASE_DATABASE_URL=${{ secrets.METABASE_MIGRATIONS_METABASE_DATABASE_URL }}" >> .env
+
+ - name: CI
+ run: make ci_metabase_migrate BIN_COMPOSER="composer" BIN_CONSOLE="php bin/console"
diff --git a/Makefile b/Makefile
index 5e996aed9..ad72a1919 100644
--- a/Makefile
+++ b/Makefile
@@ -67,6 +67,7 @@ dbinstall: ## Setup databases
make data_install
make console CMD="doctrine:database:create --env=test --if-not-exists"
make dbmigrate ARGS="--env=test"
+ make metabase_migrate ARGS="--env=test"
make data_install ARGS="--env=test"
make dbfixtures
@@ -107,6 +108,12 @@ bdtopo_migrate_redo: ## Revert db migrations for bdtopo and run them again
# Re-run migrations from there
make bdtopo_migrate
+metabase_migration: ## Generate new migration for metabase
+ ${BIN_CONSOLE} doctrine:migrations:generate --configuration ./config/packages/metabase/doctrine_migrations.yaml
+
+metabase_migrate: ## Run db migrations for metabase
+ ${BIN_CONSOLE} doctrine:migrations:migrate -n --all-or-nothing --configuration ./config/packages/metabase/doctrine_migrations.yaml ${ARGS}
+
dbshell: ## Connect to the database
docker compose exec database psql postgresql://dialog:dialog@database:5432/dialog
@@ -280,6 +287,10 @@ ci_bdtopo_migrate: ## Run CI steps for BD TOPO Migrate workflow
make composer CMD="install -n --prefer-dist"
make bdtopo_migrate
+ci_metabase_migrate: ## Run CI steps for Metabase Migrate workflow
+ make composer CMD="install -n --prefer-dist"
+ make metabase_migrate
+
ci_metabase_export: ## Export data to Metabase
scalingo login --ssh --ssh-identity ~/.ssh/id_rsa
./tools/scalingodbtunnel dialog-metabase --host-url --port 10001 & ./tools/wait-for-it.sh 127.0.0.1:10001
diff --git a/config/packages/doctrine.yaml b/config/packages/doctrine.yaml
index 0f54ffbf2..d258ac50f 100644
--- a/config/packages/doctrine.yaml
+++ b/config/packages/doctrine.yaml
@@ -36,6 +36,8 @@ doctrine:
alias: 'App\Domain'
bdtopo:
connection: bdtopo
+ metabase:
+ connection: metabase
when@test:
doctrine:
diff --git a/config/packages/metabase/doctrine_migrations.yaml b/config/packages/metabase/doctrine_migrations.yaml
new file mode 100644
index 000000000..bca5018be
--- /dev/null
+++ b/config/packages/metabase/doctrine_migrations.yaml
@@ -0,0 +1,3 @@
+migrations_paths:
+ App\Infrastructure\Persistence\Doctrine\MetabaseMigrations: 'src/Infrastructure/Persistence/Doctrine/MetabaseMigrations'
+em: metabase
diff --git a/docs/tools/metabase.md b/docs/tools/metabase.md
index caaf5efc6..ebbc535a8 100644
--- a/docs/tools/metabase.md
+++ b/docs/tools/metabase.md
@@ -18,12 +18,25 @@ La collecte des données d'indicateurs est réalisée au moyen d'une la commande
L'export Metabase peut être déclenché via [GitHub Actions](./github_actions.md) à l'aide du workflow [`metabase_export.yml`](../../.github/workflows/metabase_export.yml).
+## Tester l'export en local
+
+Vous pouvez tester l'export en local en configurant votre `.env.local` comme ceci :
+
+```bash
+METABASE_DATABASE_URL="postgresql://dialog:dialog@database:5432/dialog"
+```
+
+Lancez ensuite `make console CMD="app:metabase:export"`. Cela aura pour effet de calculer et charger les indicateurs directement dans votre base locale `dialog`.
+
+La visualisation des graphiques Metabase à partir de ces données n'est pas possible, mais vous pourrez au moins explorer les données brutes dans les tables commençant par `analytics_`.
+
### Configuration de la GitHub Action
La configuration de la GitHub Action passe par diverses variables d'environnement listées ci-dessous :
| Variable d'environnement | Configuration | Description |
|---|---|---|
-| `METABASE_EXPORT_SRC_DATABASE_URL` | [Secret](https://docs.github.com/fr/actions/security-guides/using-secrets-in-github-actions) au sens GitHub Actions | L'URL d'accès à la base de données applicative par la CI (`./tools/scalingodbtunnel dialog --host-url --port 10001`) |
-| `METABASE_EXPORT_DEST_DATABASE_URL` | Secret | L'URL d'accès à la base de données Metabase par la CI (`./tools/scalingodbtunnel dialog-metabase --host-url --port 10001`) |
+| `METABASE_MIGRATIONS_METABASE_DATABASE_URL` | [Secret](https://docs.github.com/fr/actions/security-guides/using-secrets-in-github-actions) au sens GitHub Actions | L'URL d'accès à la base de données Metabase par la CI, afin d'exécuter les migrations (`./tools/scalingodbtunnel dialog-metabase --host-url --port 10001`) |
+| `METABASE_EXPORT_DATABASE_URL` | [Secret](https://docs.github.com/fr/actions/security-guides/using-secrets-in-github-actions) au sens GitHub Actions | L'URL d'accès à la base de données applicative par la CI (`./tools/scalingodbtunnel dialog --host-url --port 10001`) |
+| `METABASE_EXPORT_METABASE_DATABASE_URL` | Secret | L'URL d'accès à la base de données Metabase par la CI (`./tools/scalingodbtunnel dialog-metabase --host-url --port 10001`) |
| `GH_SCALINGO_SSH_PRIVATE_KEY` | Secret | Clé SSH privée permettant l'accès à Scalingo par la CI |
diff --git a/phpunit.xml.dist b/phpunit.xml.dist
index 6a5374492..b9f5071dc 100644
--- a/phpunit.xml.dist
+++ b/phpunit.xml.dist
@@ -25,6 +25,7 @@
src/Infrastructure/Persistence/Doctrine/Mapping
src/Infrastructure/Persistence/Doctrine/Migrations
src/Infrastructure/Persistence/Doctrine/BdTopoMigrations
+ src/Infrastructure/Persistence/Doctrine/MetabaseMigrations
src/Infrastructure/Persistence/Doctrine/PostGIS/Event
src/Infrastructure/Adapter/CommandBus.php
src/Infrastructure/Adapter/IdFactory.php
diff --git a/src/Domain/Statistics/Repository/StatisticsRepositoryInterface.php b/src/Domain/Statistics/Repository/StatisticsRepositoryInterface.php
new file mode 100644
index 000000000..eece58d35
--- /dev/null
+++ b/src/Domain/Statistics/Repository/StatisticsRepositoryInterface.php
@@ -0,0 +1,10 @@
+addSql(
+ 'CREATE TABLE IF NOT EXISTS analytics_user_active (
+ id UUID NOT NULL,
+ uploaded_at TIMESTAMP(0),
+ last_active_at TIMESTAMP(0),
+ PRIMARY KEY(id)
+ );',
+ );
+
+ $this->addSql(
+ 'CREATE INDEX IF NOT EXISTS idx_analytics_user_active_uploaded_at
+ ON analytics_user_active (uploaded_at);',
+ );
+ }
+
+ public function down(Schema $schema): void
+ {
+ $this->addSql('DROP TABLE IF EXISTS analytics_user_active');
+ }
+}
diff --git a/src/Infrastructure/Persistence/Doctrine/Repository/Statistics/StatisticsRepository.php b/src/Infrastructure/Persistence/Doctrine/Repository/Statistics/StatisticsRepository.php
new file mode 100644
index 000000000..f2ab74dc7
--- /dev/null
+++ b/src/Infrastructure/Persistence/Doctrine/Repository/Statistics/StatisticsRepository.php
@@ -0,0 +1,42 @@
+= [uploaded_at] - 7 jours", puis en groupant sur le uploaded_at.)
+ $userRows = $this->userRepository->findAllForStatistics();
+ $this->bulkInsertUserActiveStatistics($now, $userRows);
+ }
+
+ private function bulkInsertUserActiveStatistics(\DateTimeInterface $now, array $userRows): void
+ {
+ $stmt = $this->metabaseConnection->prepare(
+ 'INSERT INTO analytics_user_active(id, uploaded_at, last_active_at)
+ VALUES (:id, (:uploadedAt)::timestamp(0), (:lastActiveAt)::timestamp(0))',
+ );
+
+ foreach ($userRows as $row) {
+ $stmt->bindValue('id', $row['uuid']);
+ $stmt->bindValue('uploadedAt', $now->format(\DateTimeInterface::ATOM));
+ $stmt->bindValue('lastActiveAt', $row['lastActiveAt']?->format(\DateTimeInterface::ATOM));
+ $stmt->execute();
+ }
+ }
+}
diff --git a/src/Infrastructure/Persistence/Doctrine/Repository/User/UserRepository.php b/src/Infrastructure/Persistence/Doctrine/Repository/User/UserRepository.php
index 6db376c5d..fb8902e30 100644
--- a/src/Infrastructure/Persistence/Doctrine/Repository/User/UserRepository.php
+++ b/src/Infrastructure/Persistence/Doctrine/Repository/User/UserRepository.php
@@ -48,4 +48,12 @@ public function countUsers(): int
->getQuery()
->getSingleScalarResult();
}
+
+ public function findAllForStatistics(): array
+ {
+ return $this->createQueryBuilder('u')
+ ->select('u.uuid, u.lastActiveAt')
+ ->getQuery()
+ ->getResult();
+ }
}
diff --git a/src/Infrastructure/Symfony/Command/RunMetabaseExportCommand.php b/src/Infrastructure/Symfony/Command/RunMetabaseExportCommand.php
index 586c09510..548560b7e 100644
--- a/src/Infrastructure/Symfony/Command/RunMetabaseExportCommand.php
+++ b/src/Infrastructure/Symfony/Command/RunMetabaseExportCommand.php
@@ -5,7 +5,7 @@
namespace App\Infrastructure\Symfony\Command;
use App\Application\DateUtilsInterface;
-use Doctrine\DBAL\Connection;
+use App\Domain\Statistics\Repository\StatisticsRepositoryInterface;
use Symfony\Component\Console\Attribute\AsCommand;
use Symfony\Component\Console\Command\Command;
use Symfony\Component\Console\Input\InputInterface;
@@ -13,15 +13,14 @@
#[AsCommand(
name: 'app:metabase:export',
- description: 'Export indicators to Metabase',
+ description: 'Export statistics to Metabase',
hidden: false,
)]
class RunMetabaseExportCommand extends Command
{
public function __construct(
private DateUtilsInterface $dateUtils,
- private Connection $connection,
- private Connection $metabaseConnection,
+ private StatisticsRepositoryInterface $statisticsRepository,
) {
parent::__construct();
}
@@ -30,34 +29,8 @@ public function execute(InputInterface $input, OutputInterface $output): int
{
$now = $this->dateUtils->getNow();
- $this->exportActiveUsers($now);
+ $this->statisticsRepository->addUserActiveStatistics($now);
return Command::SUCCESS;
}
-
- private function exportActiveUsers(\DateTimeInterface $now): void
- {
- // À chaque exécution, on ajoute la liste des dates de dernière activité pour chaque utilisateur, et la date d'exécution.
- // Dans Metabase cela permet de calculer le nombre d'utilisateurs actif au moment de chaque exécution.
- // (Par exemple avec un filtre : "[last_active_at] >= [uploaded_at] - 7 jours", puis en groupant sur le uploaded_at.)
- $this->metabaseConnection->executeQuery(
- 'CREATE TABLE IF NOT EXISTS analytics_user_active (id UUID NOT NULL, uploaded_at TIMESTAMP(0), last_active_at TIMESTAMP(0), PRIMARY KEY(id));',
- );
- $this->metabaseConnection->executeQuery(
- 'CREATE INDEX IF NOT EXISTS idx_analytics_user_active_uploaded_at ON analytics_user_active (uploaded_at);',
- );
-
- $userRows = $this->connection->fetchAllAssociative(
- 'SELECT uuid_generate_v4() AS id, last_active_at FROM "user"',
- );
-
- $stmt = $this->metabaseConnection->prepare('INSERT INTO analytics_user_active(id, uploaded_at, last_active_at) VALUES (:id, (:uploaded_at)::timestamp(0), :last_active_at)');
-
- foreach ($userRows as $row) {
- $stmt->bindValue('id', $row['id']);
- $stmt->bindValue('uploaded_at', $now->format(\DateTimeInterface::ATOM));
- $stmt->bindValue('last_active_at', $row['last_active_at']);
- $stmt->execute();
- }
- }
}