Merge branch 'main' into fix_failing_pixi_build

.docker/compose.yml

@@ -0,0 +1,12 @@
+version: '2'
+
+services:
+  qgis:
+    image: qgis/qgis:release-3_34
+    container_name: qgis
+    volumes:
+      - ../ribasim_qgis/:/tests_directory/ribasim_qgis
+    environment:
+      - CI=true
+      - DISPLAY=:99
+    tty: true

.docker/start.sh

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+set -eux
+
+docker compose -f compose.yml up -d --force-recreate --remove-orphans
+echo "Installation of the plugin Ribasim"
+docker exec -t qgis sh -c "qgis_setup.sh ribasim_qgis"
+echo "Containers are running"

.docker/stop.sh

@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+set -eux
+
+echo 'Stopping/killing containers'
+docker compose -f compose.yml kill
+docker compose -f compose.yml rm -f

.docker/test.sh

@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+set -eux
+
+docker exec -t qgis sh -c "cd /tests_directory && xvfb-run -a qgis_testrunner.sh ribasim_qgis.tests"

.dvc/.gitignore

@@ -0,0 +1,3 @@
+/config.local
+/tmp
+/cache

.dvc/config

@@ -0,0 +1,8 @@
+[core]
+    remote = minio
+['remote "minio"']  # for tracking artifacts with unreadable names
+    url = s3://ribasim/dvc
+    endpointurl = https://s3.deltares.nl
+['remote "minio_readonly"']  # for readable named models in the top-dir
+    url = s3://ribasim
+    endpointurl = https://s3.deltares.nl

.dvcignore

@@ -0,0 +1,4 @@
+# Add patterns of files dvc should ignore, which could improve
+# the performance. Learn more at
+# https://dvc.org/doc/user-guide/dvcignore
+docs

.github/workflows/core_tests.yml

@@ -37,13 +37,15 @@ jobs:
       - uses: prefix-dev/[email protected]
         with:
           pixi-version: "latest"
+      - name: Prepare pixi
+        run: pixi run install-ci
       - name: Test Ribasim Core
         run: |
           pixi run test-ribasim-core-cov
       - uses: julia-actions/julia-processcoverage@v1
         with:
           directories: core/src
-      - uses: codecov/codecov-action@v4
+      - uses: codecov/codecov-action@v5
         with:
           files: lcov.info
           token: ${{ secrets.CODECOV_TOKEN }}

.github/workflows/julia_auto_update.yml

@@ -21,7 +21,7 @@ jobs:
         run: |
           pixi run install-julia
           pixi run update-manifest-julia
-      - uses: peter-evans/create-pull-request@v6
+      - uses: peter-evans/create-pull-request@v7
         with:
           token: ${{ secrets.GITHUB_TOKEN }}
           branch: update/julia-manifest

.github/workflows/pixi_auto_update.yml

@@ -1,4 +1,8 @@
-name: Pixi auto update
+name: Update Pixi lockfile
+
+permissions:
+  contents: write
+  pull-requests: write
 
 on:
   schedule:
@@ -8,23 +12,27 @@ on:
   workflow_dispatch:
 
 jobs:
-  auto-update:
+  pixi-update:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
-        with:
-          ssh-key: ${{ secrets.SSH_PRIVATE_KEY }}
-      - uses: prefix-dev/[email protected]
+      - name: Set up pixi
+        uses: prefix-dev/[email protected]
         with:
           pixi-version: "latest"
-          cache: false
-      - name: Update pixi lock file
-        run: pixi update
-      - uses: peter-evans/create-pull-request@v6
+          run-install: false
+      - name: Update lockfiles
+        run: |
+          set -o pipefail
+          pixi update --json | pixi exec pixi-diff-to-markdown >> diff.md
+      - name: Create pull request
+        uses: peter-evans/create-pull-request@v7
         with:
           token: ${{ secrets.GITHUB_TOKEN }}
+          commit-message: Update pixi lockfile
+          title: Update pixi lockfile
+          body-path: diff.md
           branch: update/pixi-lock
-          title: Update pixi lock file
-          commit-message: "Update `pixi.lock`"
-          body: Update pixi dependencies to the latest version.
-          author: "GitHub <[email protected]>"
+          base: main
+          delete-branch: true
+          add-paths: pixi.lock

.github/workflows/pre-commit_auto_update.yml

@@ -18,7 +18,7 @@ jobs:
         with:
           python-version: "3.12"
       - uses: browniebroke/pre-commit-autoupdate-action@main
-      - uses: peter-evans/create-pull-request@v6
+      - uses: peter-evans/create-pull-request@v7
         with:
           token: ${{ secrets.GITHUB_TOKEN }}
           branch: update/pre-commit-hooks

.github/workflows/python_tests.yml

@@ -33,6 +33,6 @@ jobs:
       - name: Test Ribasim Python
         run: pixi run --environment ${{ matrix.pixi-environment }} test-ribasim-python-cov
       - name: Upload coverage to Codecov
-        uses: codecov/codecov-action@v4
+        uses: codecov/codecov-action@v5
         with:
           token: ${{ secrets.CODECOV_TOKEN }}

.github/workflows/qgis.yml

@@ -8,27 +8,21 @@ on:
     pull_request:
       paths-ignore: [".teamcity/**"]
     merge_group:
-concurrency:
-  group: ${{ github.workflow }}-${{ github.ref }}
-  cancel-in-progress: true
 jobs:
-  test:
-    name: QGIS plugin ${{ matrix.os }}
-    runs-on: ${{ matrix.os }}
-    strategy:
-      matrix:
-        os:
-          - ubuntu-latest
-          - macOS-latest
-          - windows-latest
-    steps:
-      - uses: actions/checkout@v4
-      - uses: prefix-dev/[email protected]
-        with:
-          pixi-version: "latest"
-      - name: Run tests
-        run: pixi run test-ribasim-qgis-cov
-      - name: Upload coverage to Codecov
-        uses: codecov/codecov-action@v4
-        with:
-          token: ${{ secrets.CODECOV_TOKEN }}
+    test-qgis:
+        name: "Test"
+        runs-on: ubuntu-latest
+        defaults:
+          run:
+            working-directory: .docker
+        steps:
+          - uses: actions/checkout@v4
+          - uses: prefix-dev/[email protected]
+            with:
+              pixi-version: "latest"
+          - name: Run tests
+            run: pixi run test-ribasim-qgis-docker
+          - name: Upload coverage to Codecov
+            uses: codecov/codecov-action@v5
+            with:
+              token: ${{ secrets.CODECOV_TOKEN }}

.gitignore

@@ -148,9 +148,15 @@ python/ribasim_api/tests/temp/
 report.xml
 
 # Designated working dir for working on Ribasim models
-models/
+models/*
+!models/*.dvc
 playground/output
 
 # Ruff
 .ruff_cache
 .teamcity/.idea/
+.idea/
+
+# https://github.com/Deltares/Ribasim/issues/1652
+ribasim_qgis/tests/data/database.gpkg
+ribasim_qgis/core/styles

.pre-commit-config.yaml

@@ -1,10 +1,9 @@
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.6.0
+    rev: v5.0.0
     hooks:
       - id: check-added-large-files
       - id: check-case-conflict
-      - id: check-json
       - id: check-yaml
       - id: check-toml
       - id: check-merge-conflict
@@ -14,7 +13,7 @@ repos:
         exclude: '.teamcity'
       - id: trailing-whitespace
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.5.0
+    rev: v0.7.1
     hooks:
       - id: ruff
         types_or: [python, pyi, jupyter]
@@ -26,6 +25,6 @@ repos:
     hooks:
       - id: nbstripout
   - repo: https://github.com/crate-ci/typos
-    rev: v1.22.9
+    rev: v1.26.8
     hooks:
       - id: typos

.teamcity/Ribasim/Project.kt

@@ -1,12 +1,15 @@
 package Ribasim
 
-import Ribasim.buildTypes.*
-import Ribasim.vcsRoots.*
+import Ribasim.buildTypes.GenerateTestmodels
+import Ribasim.buildTypes.Ribasim_MakeGitHubRelease
+import Ribasim.buildTypes.Ribasim_MakeQgisPlugin
+import Ribasim.vcsRoots.Ribasim
+import Ribasim_Linux.RibasimLinuxProject
+import Ribasim_Windows.RibasimWindowsProject
+import Templates.*
+import Testbench.Testbench
 import jetbrains.buildServer.configs.kotlin.Project
-import jetbrains.buildServer.configs.kotlin.projectFeatures.activeStorage
-import jetbrains.buildServer.configs.kotlin.projectFeatures.awsConnection
-import jetbrains.buildServer.configs.kotlin.projectFeatures.githubIssues
-import jetbrains.buildServer.configs.kotlin.projectFeatures.s3Storage
+import jetbrains.buildServer.configs.kotlin.projectFeatures.*
 
 object Project : Project({
 
@@ -16,31 +19,41 @@ object Project : Project({
     buildType(Ribasim_MakeQgisPlugin)
     buildType(Ribasim_MakeGitHubRelease)
 
-    template(GitHubIntegrationTemplate)
-    template(Linux_1)
-    template(Windows_1)
+    template(GithubCommitStatusIntegration)
+    template(GithubPullRequestsIntegration)
+    template(WindowsAgent)
+    template(BuildWindows)
+    template(LinuxAgent)
+    template(BuildLinux)
 
     features {
         awsConnection {
             id = "AmazonWebServicesAws"
             name = "Amazon Web Services (AWS)"
             regionName = "eu-west-3"
             credentialsType = static {
-                accessKeyId = "AKIAQBIN2MPWXSD2IZ5F"
-                secretAccessKey = "credentialsJSON:dba90026-9856-4f87-94d9-bab91f3f2d5c"
+                accessKeyId = "KwKRzscudy3GvRB8BN1Z"
+                secretAccessKey = "credentialsJSON:86cbf3e5-724c-437d-9962-7a3f429b0aa2"
                 useSessionCredentials = false
-                stsEndpoint = "https://sts.eu-west-3.amazonaws.com"
+                stsEndpoint = "https://s3-console.deltares.nl"
             }
         }
-        activeStorage {
-            id = "PROJECT_EXT_106"
-            activeStorageID = "s3_ribasim"
-        }
         githubIssues {
             id = "PROJECT_EXT_107"
             displayName = "Ribasim GitHub Issues"
             repositoryURL = "https://github.com/Deltares/Ribasim"
         }
+        s3CompatibleStorage {
+            id = "PROJECT_EXT_171"
+            accessKeyID = "KwKRzscudy3GvRB8BN1Z"
+            accessKey = "credentialsJSON:86cbf3e5-724c-437d-9962-7a3f429b0aa2"
+            endpoint = "https://s3.deltares.nl"
+            bucketName = "ribasim"
+        }
+        activeStorage {
+            id = "PROJECT_EXT_172"
+            activeStorageID = "PROJECT_EXT_171"
+        }
         s3Storage {
             id = "s3_ribasim"
             storageName = "s3"
@@ -55,6 +68,7 @@ object Project : Project({
         }
     }
 
-    subProject(Ribasim_Linux.Project)
-    subProject(Ribasim_Windows.Project)
+    subProject(RibasimLinuxProject)
+    subProject(RibasimWindowsProject)
+    subProject(Testbench)
 })

.teamcity/Ribasim/buildTypes/GenerateTestmodels.kt

@@ -1,18 +1,20 @@
 package Ribasim.buildTypes
 
+import Templates.GithubCommitStatusIntegration
 import jetbrains.buildServer.configs.kotlin.*
 import jetbrains.buildServer.configs.kotlin.buildSteps.script
 import jetbrains.buildServer.configs.kotlin.triggers.vcs
 
 object GenerateTestmodels : BuildType({
-    templates(GitHubIntegrationTemplate)
+    templates(GithubCommitStatusIntegration)
     name = "Generate Testmodels"
 
     artifactRules = """ribasim\generated_testmodels => generated_testmodels.zip"""
     publishArtifacts = PublishMode.SUCCESSFUL
 
     vcs {
         cleanCheckout = true
+        root(Ribasim.vcsRoots.Ribasim, ". => ribasim")
     }
 
     steps {