From 65fc39f3a1d88dc244aa82fc84c4f98a44885595 Mon Sep 17 00:00:00 2001 From: florimondmanca Date: Mon, 16 Dec 2024 14:36:50 +0100 Subject: [PATCH] Rework --- .github/workflows/metabase_export.yml | 4 +- .github/workflows/metabase_migrate.yml | 40 ++++++++++++++++++ Makefile | 11 +++++ config/packages/doctrine.yaml | 2 + .../metabase/doctrine_migrations.yaml | 3 ++ docs/tools/metabase.md | 17 +++++++- phpunit.xml.dist | 1 + .../StatisticsRepositoryInterface.php | 10 +++++ .../Repository/UserRepositoryInterface.php | 2 + .../Version20241216132451.php | 38 +++++++++++++++++ .../Statistics/StatisticsRepository.php | 42 +++++++++++++++++++ .../Repository/User/UserRepository.php | 8 ++++ .../Command/RunMetabaseExportCommand.php | 35 ++-------------- 13 files changed, 178 insertions(+), 35 deletions(-) create mode 100644 .github/workflows/metabase_migrate.yml create mode 100644 config/packages/metabase/doctrine_migrations.yaml create mode 100644 src/Domain/Statistics/Repository/StatisticsRepositoryInterface.php create mode 100644 src/Infrastructure/Persistence/Doctrine/MetabaseMigrations/Version20241216132451.php create mode 100644 src/Infrastructure/Persistence/Doctrine/Repository/Statistics/StatisticsRepository.php diff --git a/.github/workflows/metabase_export.yml b/.github/workflows/metabase_export.yml index 19d42e986..a17681b47 100644 --- a/.github/workflows/metabase_export.yml +++ b/.github/workflows/metabase_export.yml @@ -28,8 +28,8 @@ jobs: - name: Init CI environment variables run: | - echo "DATABASE_URL=${{ secrets.METABASE_EXPORT_SRC_DATABASE_URL }}" >> .env - echo "METABASE_DATABASE_URL=${{ secrets.METABASE_EXPORT_DEST_DATABASE_URL }}" >> .env + echo "DATABASE_URL=${{ secrets.METABASE_EXPORT_DATABASE_URL }}" >> .env + echo "METABASE_DATABASE_URL=${{ secrets.METABASE_EXPORT_METABASE_DATABASE_URL }}" >> .env - name: Run export run: make ci_metabase_export diff --git a/.github/workflows/metabase_migrate.yml b/.github/workflows/metabase_migrate.yml new file mode 100644 index 000000000..68edc0672 --- /dev/null +++ b/.github/workflows/metabase_migrate.yml @@ -0,0 +1,40 @@ +name: Metabase Migrate + +on: + workflow_dispatch: + push: + branches: + - main + paths: + - 'src/Infrastructure/Persistence/Doctrine/MetabaseMigrations/**' + +jobs: + build: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v1 + + - name: Setup PHP with PECL extension + uses: shivammathur/setup-php@v2 + with: + php-version: '8.2' + + - name: Get Composer Cache Directory + id: composer-cache + run: | + echo "dir=$(composer config cache-files-dir)" >> $GITHUB_OUTPUT + + - uses: actions/cache@v3 + with: + path: ${{ steps.composer-cache.outputs.dir }} + key: ${{ runner.os }}-composer-${{ hashFiles('**/composer.lock') }} + restore-keys: | + ${{ runner.os }}-composer- + + - name: Init environment variables + run: | + echo "METABASE_DATABASE_URL=${{ secrets.METABASE_MIGRATIONS_METABASE_DATABASE_URL }}" >> .env + + - name: CI + run: make ci_metabase_migrate BIN_COMPOSER="composer" BIN_CONSOLE="php bin/console" diff --git a/Makefile b/Makefile index 5e996aed9..ad72a1919 100644 --- a/Makefile +++ b/Makefile @@ -67,6 +67,7 @@ dbinstall: ## Setup databases make data_install make console CMD="doctrine:database:create --env=test --if-not-exists" make dbmigrate ARGS="--env=test" + make metabase_migrate ARGS="--env=test" make data_install ARGS="--env=test" make dbfixtures @@ -107,6 +108,12 @@ bdtopo_migrate_redo: ## Revert db migrations for bdtopo and run them again # Re-run migrations from there make bdtopo_migrate +metabase_migration: ## Generate new migration for metabase + ${BIN_CONSOLE} doctrine:migrations:generate --configuration ./config/packages/metabase/doctrine_migrations.yaml + +metabase_migrate: ## Run db migrations for metabase + ${BIN_CONSOLE} doctrine:migrations:migrate -n --all-or-nothing --configuration ./config/packages/metabase/doctrine_migrations.yaml ${ARGS} + dbshell: ## Connect to the database docker compose exec database psql postgresql://dialog:dialog@database:5432/dialog @@ -280,6 +287,10 @@ ci_bdtopo_migrate: ## Run CI steps for BD TOPO Migrate workflow make composer CMD="install -n --prefer-dist" make bdtopo_migrate +ci_metabase_migrate: ## Run CI steps for Metabase Migrate workflow + make composer CMD="install -n --prefer-dist" + make metabase_migrate + ci_metabase_export: ## Export data to Metabase scalingo login --ssh --ssh-identity ~/.ssh/id_rsa ./tools/scalingodbtunnel dialog-metabase --host-url --port 10001 & ./tools/wait-for-it.sh 127.0.0.1:10001 diff --git a/config/packages/doctrine.yaml b/config/packages/doctrine.yaml index 0f54ffbf2..d258ac50f 100644 --- a/config/packages/doctrine.yaml +++ b/config/packages/doctrine.yaml @@ -36,6 +36,8 @@ doctrine: alias: 'App\Domain' bdtopo: connection: bdtopo + metabase: + connection: metabase when@test: doctrine: diff --git a/config/packages/metabase/doctrine_migrations.yaml b/config/packages/metabase/doctrine_migrations.yaml new file mode 100644 index 000000000..bca5018be --- /dev/null +++ b/config/packages/metabase/doctrine_migrations.yaml @@ -0,0 +1,3 @@ +migrations_paths: + App\Infrastructure\Persistence\Doctrine\MetabaseMigrations: 'src/Infrastructure/Persistence/Doctrine/MetabaseMigrations' +em: metabase diff --git a/docs/tools/metabase.md b/docs/tools/metabase.md index caaf5efc6..ebbc535a8 100644 --- a/docs/tools/metabase.md +++ b/docs/tools/metabase.md @@ -18,12 +18,25 @@ La collecte des données d'indicateurs est réalisée au moyen d'une la commande L'export Metabase peut être déclenché via [GitHub Actions](./github_actions.md) à l'aide du workflow [`metabase_export.yml`](../../.github/workflows/metabase_export.yml). +## Tester l'export en local + +Vous pouvez tester l'export en local en configurant votre `.env.local` comme ceci : + +```bash +METABASE_DATABASE_URL="postgresql://dialog:dialog@database:5432/dialog" +``` + +Lancez ensuite `make console CMD="app:metabase:export"`. Cela aura pour effet de calculer et charger les indicateurs directement dans votre base locale `dialog`. + +La visualisation des graphiques Metabase à partir de ces données n'est pas possible, mais vous pourrez au moins explorer les données brutes dans les tables commençant par `analytics_`. + ### Configuration de la GitHub Action La configuration de la GitHub Action passe par diverses variables d'environnement listées ci-dessous : | Variable d'environnement | Configuration | Description | |---|---|---| -| `METABASE_EXPORT_SRC_DATABASE_URL` | [Secret](https://docs.github.com/fr/actions/security-guides/using-secrets-in-github-actions) au sens GitHub Actions | L'URL d'accès à la base de données applicative par la CI (`./tools/scalingodbtunnel dialog --host-url --port 10001`) | -| `METABASE_EXPORT_DEST_DATABASE_URL` | Secret | L'URL d'accès à la base de données Metabase par la CI (`./tools/scalingodbtunnel dialog-metabase --host-url --port 10001`) | +| `METABASE_MIGRATIONS_METABASE_DATABASE_URL` | [Secret](https://docs.github.com/fr/actions/security-guides/using-secrets-in-github-actions) au sens GitHub Actions | L'URL d'accès à la base de données Metabase par la CI, afin d'exécuter les migrations (`./tools/scalingodbtunnel dialog-metabase --host-url --port 10001`) | +| `METABASE_EXPORT_DATABASE_URL` | [Secret](https://docs.github.com/fr/actions/security-guides/using-secrets-in-github-actions) au sens GitHub Actions | L'URL d'accès à la base de données applicative par la CI (`./tools/scalingodbtunnel dialog --host-url --port 10001`) | +| `METABASE_EXPORT_METABASE_DATABASE_URL` | Secret | L'URL d'accès à la base de données Metabase par la CI (`./tools/scalingodbtunnel dialog-metabase --host-url --port 10001`) | | `GH_SCALINGO_SSH_PRIVATE_KEY` | Secret | Clé SSH privée permettant l'accès à Scalingo par la CI | diff --git a/phpunit.xml.dist b/phpunit.xml.dist index 6a5374492..b9f5071dc 100644 --- a/phpunit.xml.dist +++ b/phpunit.xml.dist @@ -25,6 +25,7 @@ src/Infrastructure/Persistence/Doctrine/Mapping src/Infrastructure/Persistence/Doctrine/Migrations src/Infrastructure/Persistence/Doctrine/BdTopoMigrations + src/Infrastructure/Persistence/Doctrine/MetabaseMigrations src/Infrastructure/Persistence/Doctrine/PostGIS/Event src/Infrastructure/Adapter/CommandBus.php src/Infrastructure/Adapter/IdFactory.php diff --git a/src/Domain/Statistics/Repository/StatisticsRepositoryInterface.php b/src/Domain/Statistics/Repository/StatisticsRepositoryInterface.php new file mode 100644 index 000000000..eece58d35 --- /dev/null +++ b/src/Domain/Statistics/Repository/StatisticsRepositoryInterface.php @@ -0,0 +1,10 @@ +addSql( + 'CREATE TABLE IF NOT EXISTS analytics_user_active ( + id UUID NOT NULL, + uploaded_at TIMESTAMP(0), + last_active_at TIMESTAMP(0), + PRIMARY KEY(id) + );', + ); + + $this->addSql( + 'CREATE INDEX IF NOT EXISTS idx_analytics_user_active_uploaded_at + ON analytics_user_active (uploaded_at);', + ); + } + + public function down(Schema $schema): void + { + $this->addSql('DROP TABLE IF EXISTS analytics_user_active'); + } +} diff --git a/src/Infrastructure/Persistence/Doctrine/Repository/Statistics/StatisticsRepository.php b/src/Infrastructure/Persistence/Doctrine/Repository/Statistics/StatisticsRepository.php new file mode 100644 index 000000000..f2ab74dc7 --- /dev/null +++ b/src/Infrastructure/Persistence/Doctrine/Repository/Statistics/StatisticsRepository.php @@ -0,0 +1,42 @@ += [uploaded_at] - 7 jours", puis en groupant sur le uploaded_at.) + $userRows = $this->userRepository->findAllForStatistics(); + $this->bulkInsertUserActiveStatistics($now, $userRows); + } + + private function bulkInsertUserActiveStatistics(\DateTimeInterface $now, array $userRows): void + { + $stmt = $this->metabaseConnection->prepare( + 'INSERT INTO analytics_user_active(id, uploaded_at, last_active_at) + VALUES (:id, (:uploadedAt)::timestamp(0), (:lastActiveAt)::timestamp(0))', + ); + + foreach ($userRows as $row) { + $stmt->bindValue('id', $row['uuid']); + $stmt->bindValue('uploadedAt', $now->format(\DateTimeInterface::ATOM)); + $stmt->bindValue('lastActiveAt', $row['lastActiveAt']?->format(\DateTimeInterface::ATOM)); + $stmt->execute(); + } + } +} diff --git a/src/Infrastructure/Persistence/Doctrine/Repository/User/UserRepository.php b/src/Infrastructure/Persistence/Doctrine/Repository/User/UserRepository.php index 6db376c5d..fb8902e30 100644 --- a/src/Infrastructure/Persistence/Doctrine/Repository/User/UserRepository.php +++ b/src/Infrastructure/Persistence/Doctrine/Repository/User/UserRepository.php @@ -48,4 +48,12 @@ public function countUsers(): int ->getQuery() ->getSingleScalarResult(); } + + public function findAllForStatistics(): array + { + return $this->createQueryBuilder('u') + ->select('u.uuid, u.lastActiveAt') + ->getQuery() + ->getResult(); + } } diff --git a/src/Infrastructure/Symfony/Command/RunMetabaseExportCommand.php b/src/Infrastructure/Symfony/Command/RunMetabaseExportCommand.php index 586c09510..548560b7e 100644 --- a/src/Infrastructure/Symfony/Command/RunMetabaseExportCommand.php +++ b/src/Infrastructure/Symfony/Command/RunMetabaseExportCommand.php @@ -5,7 +5,7 @@ namespace App\Infrastructure\Symfony\Command; use App\Application\DateUtilsInterface; -use Doctrine\DBAL\Connection; +use App\Domain\Statistics\Repository\StatisticsRepositoryInterface; use Symfony\Component\Console\Attribute\AsCommand; use Symfony\Component\Console\Command\Command; use Symfony\Component\Console\Input\InputInterface; @@ -13,15 +13,14 @@ #[AsCommand( name: 'app:metabase:export', - description: 'Export indicators to Metabase', + description: 'Export statistics to Metabase', hidden: false, )] class RunMetabaseExportCommand extends Command { public function __construct( private DateUtilsInterface $dateUtils, - private Connection $connection, - private Connection $metabaseConnection, + private StatisticsRepositoryInterface $statisticsRepository, ) { parent::__construct(); } @@ -30,34 +29,8 @@ public function execute(InputInterface $input, OutputInterface $output): int { $now = $this->dateUtils->getNow(); - $this->exportActiveUsers($now); + $this->statisticsRepository->addUserActiveStatistics($now); return Command::SUCCESS; } - - private function exportActiveUsers(\DateTimeInterface $now): void - { - // À chaque exécution, on ajoute la liste des dates de dernière activité pour chaque utilisateur, et la date d'exécution. - // Dans Metabase cela permet de calculer le nombre d'utilisateurs actif au moment de chaque exécution. - // (Par exemple avec un filtre : "[last_active_at] >= [uploaded_at] - 7 jours", puis en groupant sur le uploaded_at.) - $this->metabaseConnection->executeQuery( - 'CREATE TABLE IF NOT EXISTS analytics_user_active (id UUID NOT NULL, uploaded_at TIMESTAMP(0), last_active_at TIMESTAMP(0), PRIMARY KEY(id));', - ); - $this->metabaseConnection->executeQuery( - 'CREATE INDEX IF NOT EXISTS idx_analytics_user_active_uploaded_at ON analytics_user_active (uploaded_at);', - ); - - $userRows = $this->connection->fetchAllAssociative( - 'SELECT uuid_generate_v4() AS id, last_active_at FROM "user"', - ); - - $stmt = $this->metabaseConnection->prepare('INSERT INTO analytics_user_active(id, uploaded_at, last_active_at) VALUES (:id, (:uploaded_at)::timestamp(0), :last_active_at)'); - - foreach ($userRows as $row) { - $stmt->bindValue('id', $row['id']); - $stmt->bindValue('uploaded_at', $now->format(\DateTimeInterface::ATOM)); - $stmt->bindValue('last_active_at', $row['last_active_at']); - $stmt->execute(); - } - } }