Improved CI/CD

.github/workflows/merge.yaml

@@ -0,0 +1,121 @@
+name: Merge tag and release
+
+on:
+  push:
+    branches:
+      - master
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  # Don't want multiple builds running in parallel
+  cancel-in-progress: true
+
+jobs:
+  docker-build-and-push:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - run: echo "TAG=$(date +%Y%m%d).${{ github.run_number }}" >> $GITHUB_ENV
+        name: Set tag
+        id: set-tag
+
+      - run: |
+          echo -e "__git_commit__ = \"${{ github.sha }}\"\n__time__ = \"$(date)\"\n__version__ = \"${{ env.TAG }}\"" > ./app/version.py
+        name: Generate version.py before building image
+
+      - uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+        name: Login at dockerhub
+
+      - uses: docker/setup-qemu-action@v3
+        name: Setup QEMU
+
+      - uses: docker/setup-buildx-action@v3
+        name: Setup buildx
+
+      - uses: docker/build-push-action@v6
+        with:
+          file: docker/Dockerfile
+          push: true
+          tags: worthnl/notifynl-admin:latest,worthnl/notifynl-admin:${{ env.TAG }}
+        name: Docker build and push ${{ env.TAG }}
+
+      - uses: mathieudutour/[email protected]
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          custom_tag: ${{ env.TAG }}
+        name: Create git tag
+
+      - uses: softprops/action-gh-release@v2
+        with:
+          tag_name: ${{ env.TAG }}
+          make_latest: true
+        name: Create Github release
+
+    outputs:
+      tag: ${{ steps.set-tag.outputs.tag }}
+
+  helm-chart-bump:
+    runs-on: ubuntu-latest
+    needs: docker-build-and-push
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          repository: Worth-NL/notifynl-charts-private
+          ref: main
+          token: ${{ secrets.WORTHNL_PAT }}
+        name: Checkout Worth-NL/notifynl-charts-private
+
+      - uses: pietrobolcato/[email protected]
+        with:
+          config: notifynl-admin/Chart.yaml
+        name: Read Chart.yaml
+        id: yaml-read
+
+      - uses: olegsu/semver-action@v1
+        with:
+          version: ${{ steps.yaml-read.outputs['version'] }}
+        name: Chart version bump
+        id: version-bump
+
+      - uses: rmeneely/update-yaml@v1
+        with:
+          infile: notifynl-api/Chart.yaml
+          varlist: version=${{ steps.version-bump.outputs.version }},appVersion=${{ needs.docker-build-and-push.outputs.tag }}
+        name: Update Chart.yaml
+
+      - uses: offensive-vk/auto-commit-push@v7
+        with:
+          message: 🤖 notifynl-admin chart bump
+          github-token: ${{ secrets.WORTHNL_PAT }}
+
+  helm-release:
+    runs-on: ubuntu-latest
+    needs: helm-chart-bump
+    environment: Test
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          repository: Worth-NL/notifynl-charts-private
+          ref: main
+          token: ${{ secrets.WORTHNL_PAT }}
+        name: Checkout Worth-NL/notifynl-charts-private
+
+      - run: |
+          mkdir -p $HOME/.kube
+          echo "${{ secrets.K8S_CONFIG }}" > $HOME/.kube/config
+        name: Setup kubernetes config
+
+      - uses: azure/setup-helm@v4
+        name: Install helm
+
+      - run: |
+          helm version
+          helm upgrade --install notifynl-admin notifynl-admin/ -n ${{ secrets.K8S_NAMESPACE }} --reuse-values --set dockerTagOverride=${{ needs.docker-build-and-push.outputs.tag }} --wait
+        name: Deploy chart

.github/workflows/pr.yaml

@@ -0,0 +1,144 @@
+name: PR Validation
+
+on:
+  pull_request:
+    branches: ["main"]
+    types: [opened, synchronize]
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  # Don't want the tests running in parallel
+  cancel-in-progress: true
+
+jobs:
+  app-build-and-test:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ["3.11"]
+        node-version: ["20.10.0"]
+
+    steps:
+      - uses: szenius/[email protected]
+        with:
+          timezoneLinux: "Europe/Amsterdam"
+        name: Set Timezone to Europe/Amsterdam
+
+      - uses: actions/checkout@v4
+        name: Checkout repository
+
+      - uses: gerlero/apt-install@v1
+        with:
+          packages: build-essential git libcurl4-openssl-dev curl libssl-dev
+          install-recommends: false
+        name: Install OS dependencies (apt)
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+          cache: pip
+          cache-dependency-path: pyproject.toml
+        name: Install Python [${{ matrix.python-version }}] and setup cache
+
+      - uses: BSFishy/pip-action@v1
+        with:
+          requirements: requirements_for_test.txt
+        name: Install application requirements (pip)
+
+      - uses: actions/setup-node@v4
+        with:
+          node-version: ${{ matrix.node-version }}
+          cache: npm
+          cache-dependency-path: package-lock.json
+        name: Install NodeJS [${{ matrix.node-version }}] and setup cache
+
+      - run: npm ci
+        name: Install application requirements (npm)
+
+      - uses: astral-sh/ruff-action@v2
+        name: Linting (ruff)
+
+      - uses: jpetrucciani/black-check@master
+        name: Formatting (black)
+
+      - uses: BSFishy/pip-action@v1
+        with:
+          packages: |
+            pytest-md
+            pytest-emoji
+        name: Install test dependencies
+
+      # TODO: fix automated tests
+      # Connection from runner to test database is needed for tests to run
+      # or somehow sending a custom job to k8s that would run the tests and report back
+      - uses: pavelzw/pytest-action@v2
+        with:
+          verbose: false
+          job-summary: true
+        continue-on-error: true
+        name: Run tests (pytest)
+
+      - run: npm test
+        name: Run tests (node)
+
+  docker-build-and-push:
+    runs-on: ubuntu-latest
+    needs: app-build-and-test
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - run: |
+          TAG=$(date +%Y%m%d).${{ github.run_number }}.dev
+          echo "TAG=$TAG" >> $GITHUB_ENV
+          echo "tag=$TAG" >> $GITHUB_OUTPUT
+        name: Set tag
+        id: set-tag
+
+      - run: |
+          echo -e "__git_commit__ = \"${{ github.sha }}\"\n__time__ = \"$(date)\"\n__version__ = \"${{ env.TAG }}\"" > ./app/version.py
+        name: Generate version.py before building image
+
+      - uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+        name: Login at dockerhub
+
+      - uses: docker/setup-qemu-action@v3
+        name: Setup QEMU
+
+      - uses: docker/setup-buildx-action@v3
+        name: Setup buildx
+
+      - uses: docker/build-push-action@v6
+        with:
+          file: docker/Dockerfile
+          push: true
+          tags: worthnl/notifynl-admin:${{ env.TAG }}
+        name: Docker build and push ${{ env.TAG }}
+
+    outputs:
+      tag: ${{ steps.set-tag.outputs.tag }}
+
+  helm-release:
+    runs-on: ubuntu-latest
+    needs: docker-build-and-push
+    environment: Test
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          repository: Worth-NL/notifynl-charts-private
+          ref: main
+          token: ${{ secrets.WORTHNL_PAT }}
+        name: Checkout Worth-NL/notifynl-charts-private
+
+      - uses: bwvolleyball/[email protected]
+        with:
+          config: ${{ secrets.K8S_CONFIG }}
+
+      - run: |
+          helm version
+          helm upgrade --install notifynl-admin notifynl-admin/ -n ${{ secrets.K8S_NAMESPACE }} --reuse-values --set dockerTagOverride=${{ needs.docker-build-and-push.outputs.tag }} --wait
+        name: Deploy chart

.pre-commit-config.yaml

@@ -7,12 +7,12 @@ repos:
   - id: check-yaml
   - id: debug-statements
 - repo: https://github.com/charliermarsh/ruff-pre-commit
-  rev: 'v0.0.275'
+  rev: 'v0.8.3'
   hooks:
     - id: ruff
       args: [--fix, --exit-non-zero-on-fix]
 - repo: https://github.com/psf/black
-  rev: 23.10.1
+  rev: 24.10.0
   hooks:
     - id: black
       name: black (python)

app/__init__.py

@@ -16,8 +16,6 @@
     session,
     url_for,
 )
-
-from app.limiters import init_limiters
 from flask_login import LoginManager, current_user
 from flask_wtf import CSRFProtect
 from flask_wtf.csrf import CSRFError
@@ -80,6 +78,7 @@
     redact_mobile_number,
     valid_phone_number,
 )
+from app.limiters import init_limiters
 from app.models.organisation import Organisation
 from app.models.service import Service
 from app.models.user import AnonymousUser, User
@@ -130,7 +129,6 @@
 from app.utils import format_provider
 from app.utils.user_id import get_user_id_from_flask_login_session
 
-
 login_manager = LoginManager()
 csrf = CSRFProtect()
 metrics = GDSMetrics()

app/main/forms.py

@@ -1,8 +1,8 @@
-from html import escape
 import weakref
 from contextlib import suppress
 from datetime import datetime, timedelta
 from functools import partial
+from html import escape
 from itertools import chain
 from numbers import Number
 

app/main/validators.py

@@ -43,8 +43,6 @@ def __call__(self, form, field):
         if field.data == "":
             return
 
-        from flask import url_for
-
         # message = """
         #     Enter a public sector email address or
         #     <a class="govuk-link govuk-link--no-visited-state" href="{}">find out who can use Notify</a>

app/main/views/agreement.py

@@ -4,7 +4,7 @@
 from flask_login import current_user
 
 from app import current_service
-from app.limiters import limiter, RateLimit
+from app.limiters import RateLimit, limiter
 from app.main import main
 from app.main.forms import AcceptAgreementForm
 from app.models.organisation import Organisation