From c0527db5c467f902a17b67e3b6d4dc932cd64724 Mon Sep 17 00:00:00 2001 From: florimondmanca Date: Wed, 11 Dec 2024 10:54:12 +0100 Subject: [PATCH 1/4] =?UTF-8?q?Ex=C3=A9cute=20l'export=20Metabase=20depuis?= =?UTF-8?q?=20PHP=20via=20une=20commande=20Symfony?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .env | 1 + .github/workflows/ci.yml | 1 + .github/workflows/metabase_export.yml | 10 +-- Makefile | 4 +- config/packages/doctrine.yaml | 13 +++- docs/tools/metabase.md | 8 +-- src/Domain/User/User.php | 4 +- .../Doctrine/Fixtures/UserFixture.php | 6 +- .../Command/RunMetabaseExportCommand.php | 63 +++++++++++++++++++ .../AddRegulationControllerTest.php | 3 +- .../Command/RunMetabaseExportCommandTest.php | 38 +++++++++++ tools/metabase-export.sh | 16 ----- tools/metabase-export.sql | 31 --------- 13 files changed, 132 insertions(+), 66 deletions(-) create mode 100644 src/Infrastructure/Symfony/Command/RunMetabaseExportCommand.php create mode 100644 tests/Integration/Infrastructure/Symfony/Command/RunMetabaseExportCommandTest.php delete mode 100755 tools/metabase-export.sh delete mode 100644 tools/metabase-export.sql diff --git a/.env b/.env index 41a098719..b6597fc5d 100644 --- a/.env +++ b/.env @@ -22,6 +22,7 @@ APP_EUDONET_PARIS_BASE_URL=https://eudonet-partage.apps.paris.fr APP_BAC_IDF_DECREES_FILE=data/bac_idf/decrees.json APP_BAC_IDF_CITIES_FILE=data/bac_idf/cities.csv DATABASE_URL="postgresql://dialog:dialog@database:5432/dialog" +METABASE_DATABASE_URL="postgresql://dialog:dialog@database:5432/dialog" REDIS_URL="redis://redis:6379" API_ADRESSE_BASE_URL=https://api-adresse.data.gouv.fr APP_IGN_GEOCODER_BASE_URL=https://data.geopf.fr diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7da4e5736..25189801e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -61,6 +61,7 @@ jobs: run: | echo "DATABASE_URL=postgresql://dialog:dialog@localhost:5432/dialog" >> .env echo "BDTOPO_DATABASE_URL=${{ secrets.BDTOPO_DATABASE_URL }}" >> .env + echo "METABASE_DATABASE_URL=postgresql://dialog:dialog@localhost:5432/dialog" >> .env echo "REDIS_URL=redis://localhost:6379" >> .env echo "APP_STORAGE_SOURCE=memory.storage" >> .env diff --git a/.github/workflows/metabase_export.yml b/.github/workflows/metabase_export.yml index 98380a560..19d42e986 100644 --- a/.github/workflows/metabase_export.yml +++ b/.github/workflows/metabase_export.yml @@ -26,10 +26,10 @@ jobs: run: | ssh-keyscan -H ssh.osc-fr1.scalingo.com >> ~/.ssh/known_hosts + - name: Init CI environment variables + run: | + echo "DATABASE_URL=${{ secrets.METABASE_EXPORT_SRC_DATABASE_URL }}" >> .env + echo "METABASE_DATABASE_URL=${{ secrets.METABASE_EXPORT_DEST_DATABASE_URL }}" >> .env + - name: Run export run: make ci_metabase_export - env: - METABASE_SRC_APP: ${{ vars.METABASE_EXPORT_SRC_APP }} - METABASE_SRC_DATABASE_URL: ${{ secrets.METABASE_EXPORT_SRC_DATABASE_URL }} - METABASE_DEST_APP: ${{ vars.METABASE_EXPORT_DEST_APP }} - METABASE_DEST_DATABASE_URL: ${{ secrets.METABASE_EXPORT_DEST_DATABASE_URL }} diff --git a/Makefile b/Makefile index 8398fbd4b..5e996aed9 100644 --- a/Makefile +++ b/Makefile @@ -282,8 +282,8 @@ ci_bdtopo_migrate: ## Run CI steps for BD TOPO Migrate workflow ci_metabase_export: ## Export data to Metabase scalingo login --ssh --ssh-identity ~/.ssh/id_rsa - ./tools/scalingodbtunnel ${METABASE_DEST_APP} --host-url --port 10001 & ./tools/wait-for-it.sh 127.0.0.1:10001 - ./tools/metabase-export.sh ${METABASE_SRC_DATABASE_URL} ${METABASE_DEST_DATABASE_URL} + ./tools/scalingodbtunnel dialog-metabase --host-url --port 10001 & ./tools/wait-for-it.sh 127.0.0.1:10001 + make console CMD="app:metabase:export" ## ## ---------------- diff --git a/config/packages/doctrine.yaml b/config/packages/doctrine.yaml index 5ccb4a623..0f54ffbf2 100644 --- a/config/packages/doctrine.yaml +++ b/config/packages/doctrine.yaml @@ -8,7 +8,10 @@ doctrine: schema_manager_factory: 'doctrine.dbal.default_schema_manager_factory' use_savepoints: true bdtopo: - url: '%env(BDTOPO_DATABASE_URL)%' + url: '%env(resolve:BDTOPO_DATABASE_URL)%' + use_savepoints: true + metabase: + url: '%env(resolve:METABASE_DATABASE_URL)%' use_savepoints: true orm: auto_generate_proxy_classes: true @@ -37,8 +40,12 @@ doctrine: when@test: doctrine: dbal: - # "TEST_TOKEN" is typically set by ParaTest - dbname_suffix: '_test%env(default::TEST_TOKEN)%' + connections: + default: + # "TEST_TOKEN" is typically set by ParaTest + dbname_suffix: '_test%env(default::TEST_TOKEN)%' + metabase: + dbname_suffix: '_test%env(default::TEST_TOKEN)%' when@prod: doctrine: diff --git a/docs/tools/metabase.md b/docs/tools/metabase.md index 087bbe7ad..caaf5efc6 100644 --- a/docs/tools/metabase.md +++ b/docs/tools/metabase.md @@ -12,7 +12,7 @@ Le Metabase de DiaLog est hébergé sur Scalingo sous l'application `dialog-meta Cette application dispose de sa propre base de données où nous stockons les données nécessaires au calcul des indicateurs, conformément aux [recommendations Beta](https://doc.incubateur.net/communaute/les-outils-de-la-communaute/autres-services/metabase/metabase#connecter-metabase-a-une-base-de-donnees-anonymisee) -La collecte des données d'indicateurs est réalisée au moyen d'un [script](../../tools/metabase-export.sh). Ce script exécute des requêtes SQL depuis la base Metabase vers la base applicative. Pour cela un utilisateur `dialog_metabase` avec droits en lecture seule a été créé sur la base applicative (identifiants dans le Vaultwarden de l'équipe DiaLog). +La collecte des données d'indicateurs est réalisée au moyen d'une la commande Symfony `app:metabase:export`. Cette commande rassemble les données sources (requêtes à la base de données, requêtes HTTP, ou autres opérations...) puis les upload vers la base de données PostgreSQL de l'instance Metabase. ## Lancer l'export depuis GitHub Actions @@ -24,8 +24,6 @@ La configuration de la GitHub Action passe par diverses variables d'environnemen | Variable d'environnement | Configuration | Description | |---|---|---| -| `METABASE_EXPORT_SRC_APP` | [Variable](https://docs.github.com/fr/actions/learn-github-actions/variables) au sens GitHub Actions | `dialog` (pour la production) | -| `METABASE_EXPORT_SRC_DATABASE_URL` | [Secret](https://docs.github.com/fr/actions/security-guides/using-secrets-in-github-actions) au sens GitHub Actions | L'URL d'accès à la base de données applicative par la DB Metabase : utiliser la `METABASE_EXPORT_SRC_DATABASE_URL` de l'app `dialog` | -| `METABASE_EXPORT_DEST_APP` | Variable | `dialog-metabase` | -| `METABASE_EXPORT_DEST_DATABASE_URL` | Secret | L'URL d'accès à la base de données Metabase par la CI (`./tools/scalingodbtunnel dialog-metabase --host-url --port 10001`) | +| `METABASE_EXPORT_SRC_DATABASE_URL` | [Secret](https://docs.github.com/fr/actions/security-guides/using-secrets-in-github-actions) au sens GitHub Actions | L'URL d'accès à la base de données applicative par la CI (`./tools/scalingodbtunnel dialog --host-url --port 10001`) | +| `METABASE_EXPORT_DEST_DATABASE_URL` | Secret | L'URL d'accès à la base de données Metabase par la CI (`./tools/scalingodbtunnel dialog-metabase --host-url --port 10001`) | | `GH_SCALINGO_SSH_PRIVATE_KEY` | Secret | Clé SSH privée permettant l'accès à Scalingo par la CI | diff --git a/src/Domain/User/User.php b/src/Domain/User/User.php index 57d595f67..4f30109f9 100644 --- a/src/Domain/User/User.php +++ b/src/Domain/User/User.php @@ -88,9 +88,11 @@ public function getLastActiveAt(): ?\DateTimeInterface return $this->lastActiveAt; } - public function setLastActiveAt(\DateTimeInterface $date): void + public function setLastActiveAt(\DateTimeInterface $date): self { $this->lastActiveAt = $date; + + return $this; } public function __toString(): string diff --git a/src/Infrastructure/Persistence/Doctrine/Fixtures/UserFixture.php b/src/Infrastructure/Persistence/Doctrine/Fixtures/UserFixture.php index b5c6cb1d5..dfcc90a81 100644 --- a/src/Infrastructure/Persistence/Doctrine/Fixtures/UserFixture.php +++ b/src/Infrastructure/Persistence/Doctrine/Fixtures/UserFixture.php @@ -23,14 +23,16 @@ public function load(ObjectManager $manager): void ->setEmail(self::MAIN_ORG_USER_EMAIL) ->setPassword(self::PASSWORD) ->setRoles([UserRolesEnum::ROLE_USER->value]) - ->setRegistrationDate(new \DateTimeImmutable('2024-03-01')); + ->setRegistrationDate(new \DateTimeImmutable('2024-03-01')) + ->setLastActiveAt(new \DateTimeImmutable('2024-06-07')); $mainOtherAdmin = (new User('5bc831a3-7a09-44e9-aefa-5ce3588dac33')) ->setFullName('Mathieu FERNANDEZ') ->setEmail(self::MAIN_ORG_ADMIN_EMAIL) ->setPassword(self::PASSWORD) ->setRoles([UserRolesEnum::ROLE_SUPER_ADMIN->value]) - ->setRegistrationDate(new \DateTimeImmutable('2024-04-02')); + ->setRegistrationDate(new \DateTimeImmutable('2024-04-02')) + ->setLastActiveAt(new \DateTimeImmutable('2024-06-08')); $otherOrgUser = (new User('d47badd9-989e-472b-a80e-9df642e93880')) ->setFullName('Florimond MANCA') diff --git a/src/Infrastructure/Symfony/Command/RunMetabaseExportCommand.php b/src/Infrastructure/Symfony/Command/RunMetabaseExportCommand.php new file mode 100644 index 000000000..586c09510 --- /dev/null +++ b/src/Infrastructure/Symfony/Command/RunMetabaseExportCommand.php @@ -0,0 +1,63 @@ +dateUtils->getNow(); + + $this->exportActiveUsers($now); + + return Command::SUCCESS; + } + + private function exportActiveUsers(\DateTimeInterface $now): void + { + // À chaque exécution, on ajoute la liste des dates de dernière activité pour chaque utilisateur, et la date d'exécution. + // Dans Metabase cela permet de calculer le nombre d'utilisateurs actif au moment de chaque exécution. + // (Par exemple avec un filtre : "[last_active_at] >= [uploaded_at] - 7 jours", puis en groupant sur le uploaded_at.) + $this->metabaseConnection->executeQuery( + 'CREATE TABLE IF NOT EXISTS analytics_user_active (id UUID NOT NULL, uploaded_at TIMESTAMP(0), last_active_at TIMESTAMP(0), PRIMARY KEY(id));', + ); + $this->metabaseConnection->executeQuery( + 'CREATE INDEX IF NOT EXISTS idx_analytics_user_active_uploaded_at ON analytics_user_active (uploaded_at);', + ); + + $userRows = $this->connection->fetchAllAssociative( + 'SELECT uuid_generate_v4() AS id, last_active_at FROM "user"', + ); + + $stmt = $this->metabaseConnection->prepare('INSERT INTO analytics_user_active(id, uploaded_at, last_active_at) VALUES (:id, (:uploaded_at)::timestamp(0), :last_active_at)'); + + foreach ($userRows as $row) { + $stmt->bindValue('id', $row['id']); + $stmt->bindValue('uploaded_at', $now->format(\DateTimeInterface::ATOM)); + $stmt->bindValue('last_active_at', $row['last_active_at']); + $stmt->execute(); + } + } +} diff --git a/tests/Integration/Infrastructure/Controller/Regulation/AddRegulationControllerTest.php b/tests/Integration/Infrastructure/Controller/Regulation/AddRegulationControllerTest.php index 803b2d865..118da7d65 100644 --- a/tests/Integration/Infrastructure/Controller/Regulation/AddRegulationControllerTest.php +++ b/tests/Integration/Infrastructure/Controller/Regulation/AddRegulationControllerTest.php @@ -38,7 +38,7 @@ public function testAdd(): void /** @var UserRepositoryInterface */ $userRepository = static::getContainer()->get(UserRepositoryInterface::class); - $this->assertNull($userRepository->findOneByEmail($email)->getLastActiveAt()); + $this->assertEquals(new \DateTimeImmutable('2024-06-07'), $userRepository->findOneByEmail($email)->getLastActiveAt()); // Get the raw values. $values = $form->getPhpValues(); @@ -56,6 +56,7 @@ public function testAdd(): void $crawler = $client->request($form->getMethod(), $form->getUri(), $values, $form->getPhpFiles()); $this->assertResponseStatusCodeSame(303); + // Filled with DateUtilsMock::getNow() $this->assertEquals(new \DateTimeImmutable('2023-06-09'), $userRepository->findOneByEmail($email)->getLastActiveAt()); $client->followRedirect(); diff --git a/tests/Integration/Infrastructure/Symfony/Command/RunMetabaseExportCommandTest.php b/tests/Integration/Infrastructure/Symfony/Command/RunMetabaseExportCommandTest.php new file mode 100644 index 000000000..8c3102161 --- /dev/null +++ b/tests/Integration/Infrastructure/Symfony/Command/RunMetabaseExportCommandTest.php @@ -0,0 +1,38 @@ +get(RunMetabaseExportCommand::class); + $commandTester = new CommandTester($command); + $commandTester->execute([]); + $commandTester->assertCommandIsSuccessful(); + + /** @var \Doctrine\DBAL\Connection */ + $metabaseConnection = $container->get('doctrine.dbal.metabase_connection'); + $rows = $metabaseConnection->fetchAllAssociative('SELECT * FROM analytics_user_active'); + $this->assertCount(3, $rows); + $this->assertEquals(['id', 'uploaded_at', 'last_active_at'], array_keys($rows[0])); + + $this->assertSame('2023-06-09 00:00:00', $rows[0]['uploaded_at']); + $this->assertSame('2024-06-07 00:00:00', $rows[0]['last_active_at']) + ; + $this->assertSame('2023-06-09 00:00:00', $rows[1]['uploaded_at']); + $this->assertSame('2024-06-08 00:00:00', $rows[1]['last_active_at']); + + $this->assertSame('2023-06-09 00:00:00', $rows[2]['uploaded_at']); + $this->assertSame(null, $rows[2]['last_active_at']); + } +} diff --git a/tools/metabase-export.sh b/tools/metabase-export.sh deleted file mode 100755 index 4e56c92bd..000000000 --- a/tools/metabase-export.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/bash -# Inspiré de : https://doc.incubateur.net/communaute/les-outils-de-la-communaute/autres-services/metabase/metabase#connecter-metabase-a-une-base-de-donnees-anonymisee -set -euxo pipefail - -# URL de la DB DiaLog du point de vue de la DB Metabase (sera utilisé avec dblink) -SRC_DATABASE_URL=$1 - -# URL de la DB Metabase du point de vue de ce script -DEST_DATABASE_URL=$2 - -export PGOPTIONS="-c custom.src_database_url=${SRC_DATABASE_URL}" - -# ON_ERROR_STOP=1 s'assure que cette commande échoue (return code != 0) si le script SQL a des statements qui échouent. -# (Par défaut avec -f on a toujours un return code 0 et un statement en échec n'empêche pas les suivants de s'exécuter.) -# https://engineering.nordeus.com/psql-exit-on-first-error/ -psql $DEST_DATABASE_URL -v ON_ERROR_STOP=1 -f ./tools/metabase-export.sql diff --git a/tools/metabase-export.sql b/tools/metabase-export.sql deleted file mode 100644 index 4b2e50c04..000000000 --- a/tools/metabase-export.sql +++ /dev/null @@ -1,31 +0,0 @@ --- metabase-export.sql --- Ce script est conçu pour être exécuté sur la base de données PostgreSQL de l'instance Metabase (destination). --- Il consiste à extraire des données de la base applicative (source) pour les charger dans des tables Metabase. - --- Configuration générale -CREATE EXTENSION IF NOT EXISTS "uuid-ossp"; - --- CONNEXION À LA DB APPLICATIVE (source) --- Voir : https://www.postgresql.org/docs/current/contrib-dblink-connect.html -CREATE EXTENSION IF NOT EXISTS dblink; -SELECT dblink_connect('src', current_setting('custom.src_database_url')); - --- COLLECTE DES DONNÉES D'INDICATEURS - --- # Utilisateurs actifs --- À chaque exécution, on ajoute la liste des dates de dernière activité pour chaque utilisateur, assortie de la date d'exécution. --- Dans Metabase cela permet de calculer le nombre d'utilisateurs actif au moment de chaque exécution. --- (Par exemple avec un filtre : "[last_active_at] >= [uploaded_at] - 7 jours", puis en groupant sur le uploaded_at.) -CREATE TABLE IF NOT EXISTS analytics_user_active (id UUID NOT NULL, uploaded_at TIMESTAMP(0), last_active_at TIMESTAMP(0), PRIMARY KEY(id)); -CREATE INDEX IF NOT EXISTS idx_analytics_user_active_uploaded_at ON analytics_user_active (uploaded_at); - -WITH params AS ( - -- Calculé 1 bonne fois pour toute pour que toutes les lignes utilisent exactement la même valeur à des fins de groupement dans Metabase - SELECT NOW() as current_date -) -INSERT INTO analytics_user_active(id, uploaded_at, last_active_at) -SELECT uuid_generate_v4() AS id, p.current_date AS uploaded_at, u.last_active_at AS last_active_at -FROM - dblink('src', 'SELECT last_active_at FROM "user"') AS u(last_active_at TIMESTAMP(0) WITH TIME ZONE), - params AS p -; From 65fc39f3a1d88dc244aa82fc84c4f98a44885595 Mon Sep 17 00:00:00 2001 From: florimondmanca Date: Mon, 16 Dec 2024 14:36:50 +0100 Subject: [PATCH 2/4] Rework --- .github/workflows/metabase_export.yml | 4 +- .github/workflows/metabase_migrate.yml | 40 ++++++++++++++++++ Makefile | 11 +++++ config/packages/doctrine.yaml | 2 + .../metabase/doctrine_migrations.yaml | 3 ++ docs/tools/metabase.md | 17 +++++++- phpunit.xml.dist | 1 + .../StatisticsRepositoryInterface.php | 10 +++++ .../Repository/UserRepositoryInterface.php | 2 + .../Version20241216132451.php | 38 +++++++++++++++++ .../Statistics/StatisticsRepository.php | 42 +++++++++++++++++++ .../Repository/User/UserRepository.php | 8 ++++ .../Command/RunMetabaseExportCommand.php | 35 ++-------------- 13 files changed, 178 insertions(+), 35 deletions(-) create mode 100644 .github/workflows/metabase_migrate.yml create mode 100644 config/packages/metabase/doctrine_migrations.yaml create mode 100644 src/Domain/Statistics/Repository/StatisticsRepositoryInterface.php create mode 100644 src/Infrastructure/Persistence/Doctrine/MetabaseMigrations/Version20241216132451.php create mode 100644 src/Infrastructure/Persistence/Doctrine/Repository/Statistics/StatisticsRepository.php diff --git a/.github/workflows/metabase_export.yml b/.github/workflows/metabase_export.yml index 19d42e986..a17681b47 100644 --- a/.github/workflows/metabase_export.yml +++ b/.github/workflows/metabase_export.yml @@ -28,8 +28,8 @@ jobs: - name: Init CI environment variables run: | - echo "DATABASE_URL=${{ secrets.METABASE_EXPORT_SRC_DATABASE_URL }}" >> .env - echo "METABASE_DATABASE_URL=${{ secrets.METABASE_EXPORT_DEST_DATABASE_URL }}" >> .env + echo "DATABASE_URL=${{ secrets.METABASE_EXPORT_DATABASE_URL }}" >> .env + echo "METABASE_DATABASE_URL=${{ secrets.METABASE_EXPORT_METABASE_DATABASE_URL }}" >> .env - name: Run export run: make ci_metabase_export diff --git a/.github/workflows/metabase_migrate.yml b/.github/workflows/metabase_migrate.yml new file mode 100644 index 000000000..68edc0672 --- /dev/null +++ b/.github/workflows/metabase_migrate.yml @@ -0,0 +1,40 @@ +name: Metabase Migrate + +on: + workflow_dispatch: + push: + branches: + - main + paths: + - 'src/Infrastructure/Persistence/Doctrine/MetabaseMigrations/**' + +jobs: + build: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v1 + + - name: Setup PHP with PECL extension + uses: shivammathur/setup-php@v2 + with: + php-version: '8.2' + + - name: Get Composer Cache Directory + id: composer-cache + run: | + echo "dir=$(composer config cache-files-dir)" >> $GITHUB_OUTPUT + + - uses: actions/cache@v3 + with: + path: ${{ steps.composer-cache.outputs.dir }} + key: ${{ runner.os }}-composer-${{ hashFiles('**/composer.lock') }} + restore-keys: | + ${{ runner.os }}-composer- + + - name: Init environment variables + run: | + echo "METABASE_DATABASE_URL=${{ secrets.METABASE_MIGRATIONS_METABASE_DATABASE_URL }}" >> .env + + - name: CI + run: make ci_metabase_migrate BIN_COMPOSER="composer" BIN_CONSOLE="php bin/console" diff --git a/Makefile b/Makefile index 5e996aed9..ad72a1919 100644 --- a/Makefile +++ b/Makefile @@ -67,6 +67,7 @@ dbinstall: ## Setup databases make data_install make console CMD="doctrine:database:create --env=test --if-not-exists" make dbmigrate ARGS="--env=test" + make metabase_migrate ARGS="--env=test" make data_install ARGS="--env=test" make dbfixtures @@ -107,6 +108,12 @@ bdtopo_migrate_redo: ## Revert db migrations for bdtopo and run them again # Re-run migrations from there make bdtopo_migrate +metabase_migration: ## Generate new migration for metabase + ${BIN_CONSOLE} doctrine:migrations:generate --configuration ./config/packages/metabase/doctrine_migrations.yaml + +metabase_migrate: ## Run db migrations for metabase + ${BIN_CONSOLE} doctrine:migrations:migrate -n --all-or-nothing --configuration ./config/packages/metabase/doctrine_migrations.yaml ${ARGS} + dbshell: ## Connect to the database docker compose exec database psql postgresql://dialog:dialog@database:5432/dialog @@ -280,6 +287,10 @@ ci_bdtopo_migrate: ## Run CI steps for BD TOPO Migrate workflow make composer CMD="install -n --prefer-dist" make bdtopo_migrate +ci_metabase_migrate: ## Run CI steps for Metabase Migrate workflow + make composer CMD="install -n --prefer-dist" + make metabase_migrate + ci_metabase_export: ## Export data to Metabase scalingo login --ssh --ssh-identity ~/.ssh/id_rsa ./tools/scalingodbtunnel dialog-metabase --host-url --port 10001 & ./tools/wait-for-it.sh 127.0.0.1:10001 diff --git a/config/packages/doctrine.yaml b/config/packages/doctrine.yaml index 0f54ffbf2..d258ac50f 100644 --- a/config/packages/doctrine.yaml +++ b/config/packages/doctrine.yaml @@ -36,6 +36,8 @@ doctrine: alias: 'App\Domain' bdtopo: connection: bdtopo + metabase: + connection: metabase when@test: doctrine: diff --git a/config/packages/metabase/doctrine_migrations.yaml b/config/packages/metabase/doctrine_migrations.yaml new file mode 100644 index 000000000..bca5018be --- /dev/null +++ b/config/packages/metabase/doctrine_migrations.yaml @@ -0,0 +1,3 @@ +migrations_paths: + App\Infrastructure\Persistence\Doctrine\MetabaseMigrations: 'src/Infrastructure/Persistence/Doctrine/MetabaseMigrations' +em: metabase diff --git a/docs/tools/metabase.md b/docs/tools/metabase.md index caaf5efc6..ebbc535a8 100644 --- a/docs/tools/metabase.md +++ b/docs/tools/metabase.md @@ -18,12 +18,25 @@ La collecte des données d'indicateurs est réalisée au moyen d'une la commande L'export Metabase peut être déclenché via [GitHub Actions](./github_actions.md) à l'aide du workflow [`metabase_export.yml`](../../.github/workflows/metabase_export.yml). +## Tester l'export en local + +Vous pouvez tester l'export en local en configurant votre `.env.local` comme ceci : + +```bash +METABASE_DATABASE_URL="postgresql://dialog:dialog@database:5432/dialog" +``` + +Lancez ensuite `make console CMD="app:metabase:export"`. Cela aura pour effet de calculer et charger les indicateurs directement dans votre base locale `dialog`. + +La visualisation des graphiques Metabase à partir de ces données n'est pas possible, mais vous pourrez au moins explorer les données brutes dans les tables commençant par `analytics_`. + ### Configuration de la GitHub Action La configuration de la GitHub Action passe par diverses variables d'environnement listées ci-dessous : | Variable d'environnement | Configuration | Description | |---|---|---| -| `METABASE_EXPORT_SRC_DATABASE_URL` | [Secret](https://docs.github.com/fr/actions/security-guides/using-secrets-in-github-actions) au sens GitHub Actions | L'URL d'accès à la base de données applicative par la CI (`./tools/scalingodbtunnel dialog --host-url --port 10001`) | -| `METABASE_EXPORT_DEST_DATABASE_URL` | Secret | L'URL d'accès à la base de données Metabase par la CI (`./tools/scalingodbtunnel dialog-metabase --host-url --port 10001`) | +| `METABASE_MIGRATIONS_METABASE_DATABASE_URL` | [Secret](https://docs.github.com/fr/actions/security-guides/using-secrets-in-github-actions) au sens GitHub Actions | L'URL d'accès à la base de données Metabase par la CI, afin d'exécuter les migrations (`./tools/scalingodbtunnel dialog-metabase --host-url --port 10001`) | +| `METABASE_EXPORT_DATABASE_URL` | [Secret](https://docs.github.com/fr/actions/security-guides/using-secrets-in-github-actions) au sens GitHub Actions | L'URL d'accès à la base de données applicative par la CI (`./tools/scalingodbtunnel dialog --host-url --port 10001`) | +| `METABASE_EXPORT_METABASE_DATABASE_URL` | Secret | L'URL d'accès à la base de données Metabase par la CI (`./tools/scalingodbtunnel dialog-metabase --host-url --port 10001`) | | `GH_SCALINGO_SSH_PRIVATE_KEY` | Secret | Clé SSH privée permettant l'accès à Scalingo par la CI | diff --git a/phpunit.xml.dist b/phpunit.xml.dist index 6a5374492..b9f5071dc 100644 --- a/phpunit.xml.dist +++ b/phpunit.xml.dist @@ -25,6 +25,7 @@ src/Infrastructure/Persistence/Doctrine/Mapping src/Infrastructure/Persistence/Doctrine/Migrations src/Infrastructure/Persistence/Doctrine/BdTopoMigrations + src/Infrastructure/Persistence/Doctrine/MetabaseMigrations src/Infrastructure/Persistence/Doctrine/PostGIS/Event src/Infrastructure/Adapter/CommandBus.php src/Infrastructure/Adapter/IdFactory.php diff --git a/src/Domain/Statistics/Repository/StatisticsRepositoryInterface.php b/src/Domain/Statistics/Repository/StatisticsRepositoryInterface.php new file mode 100644 index 000000000..eece58d35 --- /dev/null +++ b/src/Domain/Statistics/Repository/StatisticsRepositoryInterface.php @@ -0,0 +1,10 @@ +addSql( + 'CREATE TABLE IF NOT EXISTS analytics_user_active ( + id UUID NOT NULL, + uploaded_at TIMESTAMP(0), + last_active_at TIMESTAMP(0), + PRIMARY KEY(id) + );', + ); + + $this->addSql( + 'CREATE INDEX IF NOT EXISTS idx_analytics_user_active_uploaded_at + ON analytics_user_active (uploaded_at);', + ); + } + + public function down(Schema $schema): void + { + $this->addSql('DROP TABLE IF EXISTS analytics_user_active'); + } +} diff --git a/src/Infrastructure/Persistence/Doctrine/Repository/Statistics/StatisticsRepository.php b/src/Infrastructure/Persistence/Doctrine/Repository/Statistics/StatisticsRepository.php new file mode 100644 index 000000000..f2ab74dc7 --- /dev/null +++ b/src/Infrastructure/Persistence/Doctrine/Repository/Statistics/StatisticsRepository.php @@ -0,0 +1,42 @@ += [uploaded_at] - 7 jours", puis en groupant sur le uploaded_at.) + $userRows = $this->userRepository->findAllForStatistics(); + $this->bulkInsertUserActiveStatistics($now, $userRows); + } + + private function bulkInsertUserActiveStatistics(\DateTimeInterface $now, array $userRows): void + { + $stmt = $this->metabaseConnection->prepare( + 'INSERT INTO analytics_user_active(id, uploaded_at, last_active_at) + VALUES (:id, (:uploadedAt)::timestamp(0), (:lastActiveAt)::timestamp(0))', + ); + + foreach ($userRows as $row) { + $stmt->bindValue('id', $row['uuid']); + $stmt->bindValue('uploadedAt', $now->format(\DateTimeInterface::ATOM)); + $stmt->bindValue('lastActiveAt', $row['lastActiveAt']?->format(\DateTimeInterface::ATOM)); + $stmt->execute(); + } + } +} diff --git a/src/Infrastructure/Persistence/Doctrine/Repository/User/UserRepository.php b/src/Infrastructure/Persistence/Doctrine/Repository/User/UserRepository.php index 6db376c5d..fb8902e30 100644 --- a/src/Infrastructure/Persistence/Doctrine/Repository/User/UserRepository.php +++ b/src/Infrastructure/Persistence/Doctrine/Repository/User/UserRepository.php @@ -48,4 +48,12 @@ public function countUsers(): int ->getQuery() ->getSingleScalarResult(); } + + public function findAllForStatistics(): array + { + return $this->createQueryBuilder('u') + ->select('u.uuid, u.lastActiveAt') + ->getQuery() + ->getResult(); + } } diff --git a/src/Infrastructure/Symfony/Command/RunMetabaseExportCommand.php b/src/Infrastructure/Symfony/Command/RunMetabaseExportCommand.php index 586c09510..548560b7e 100644 --- a/src/Infrastructure/Symfony/Command/RunMetabaseExportCommand.php +++ b/src/Infrastructure/Symfony/Command/RunMetabaseExportCommand.php @@ -5,7 +5,7 @@ namespace App\Infrastructure\Symfony\Command; use App\Application\DateUtilsInterface; -use Doctrine\DBAL\Connection; +use App\Domain\Statistics\Repository\StatisticsRepositoryInterface; use Symfony\Component\Console\Attribute\AsCommand; use Symfony\Component\Console\Command\Command; use Symfony\Component\Console\Input\InputInterface; @@ -13,15 +13,14 @@ #[AsCommand( name: 'app:metabase:export', - description: 'Export indicators to Metabase', + description: 'Export statistics to Metabase', hidden: false, )] class RunMetabaseExportCommand extends Command { public function __construct( private DateUtilsInterface $dateUtils, - private Connection $connection, - private Connection $metabaseConnection, + private StatisticsRepositoryInterface $statisticsRepository, ) { parent::__construct(); } @@ -30,34 +29,8 @@ public function execute(InputInterface $input, OutputInterface $output): int { $now = $this->dateUtils->getNow(); - $this->exportActiveUsers($now); + $this->statisticsRepository->addUserActiveStatistics($now); return Command::SUCCESS; } - - private function exportActiveUsers(\DateTimeInterface $now): void - { - // À chaque exécution, on ajoute la liste des dates de dernière activité pour chaque utilisateur, et la date d'exécution. - // Dans Metabase cela permet de calculer le nombre d'utilisateurs actif au moment de chaque exécution. - // (Par exemple avec un filtre : "[last_active_at] >= [uploaded_at] - 7 jours", puis en groupant sur le uploaded_at.) - $this->metabaseConnection->executeQuery( - 'CREATE TABLE IF NOT EXISTS analytics_user_active (id UUID NOT NULL, uploaded_at TIMESTAMP(0), last_active_at TIMESTAMP(0), PRIMARY KEY(id));', - ); - $this->metabaseConnection->executeQuery( - 'CREATE INDEX IF NOT EXISTS idx_analytics_user_active_uploaded_at ON analytics_user_active (uploaded_at);', - ); - - $userRows = $this->connection->fetchAllAssociative( - 'SELECT uuid_generate_v4() AS id, last_active_at FROM "user"', - ); - - $stmt = $this->metabaseConnection->prepare('INSERT INTO analytics_user_active(id, uploaded_at, last_active_at) VALUES (:id, (:uploaded_at)::timestamp(0), :last_active_at)'); - - foreach ($userRows as $row) { - $stmt->bindValue('id', $row['id']); - $stmt->bindValue('uploaded_at', $now->format(\DateTimeInterface::ATOM)); - $stmt->bindValue('last_active_at', $row['last_active_at']); - $stmt->execute(); - } - } } From fce2a1e13d040d6e1ce2575b8a4f6e3c65579b03 Mon Sep 17 00:00:00 2001 From: florimondmanca Date: Tue, 17 Dec 2024 09:58:19 +0100 Subject: [PATCH 3/4] Feedback --- config/packages/doctrine.yaml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/config/packages/doctrine.yaml b/config/packages/doctrine.yaml index d258ac50f..844cd9555 100644 --- a/config/packages/doctrine.yaml +++ b/config/packages/doctrine.yaml @@ -8,11 +8,9 @@ doctrine: schema_manager_factory: 'doctrine.dbal.default_schema_manager_factory' use_savepoints: true bdtopo: - url: '%env(resolve:BDTOPO_DATABASE_URL)%' - use_savepoints: true + url: '%env(BDTOPO_DATABASE_URL)%' metabase: - url: '%env(resolve:METABASE_DATABASE_URL)%' - use_savepoints: true + url: '%env(METABASE_DATABASE_URL)%' orm: auto_generate_proxy_classes: true default_entity_manager: default From 02c9467a0027cf3d11b4ce1a14ada1e00810c762 Mon Sep 17 00:00:00 2001 From: florimondmanca Date: Tue, 17 Dec 2024 10:18:05 +0100 Subject: [PATCH 4/4] Revert use_savepoints --- config/packages/doctrine.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/config/packages/doctrine.yaml b/config/packages/doctrine.yaml index 844cd9555..070489211 100644 --- a/config/packages/doctrine.yaml +++ b/config/packages/doctrine.yaml @@ -9,8 +9,10 @@ doctrine: use_savepoints: true bdtopo: url: '%env(BDTOPO_DATABASE_URL)%' + use_savepoints: true metabase: url: '%env(METABASE_DATABASE_URL)%' + use_savepoints: true orm: auto_generate_proxy_classes: true default_entity_manager: default