Adding support for multiple databases

.github/workflows/create-release-n-publish.yml

@@ -20,14 +20,9 @@ jobs:
         with:
           python-version: "3.10"
 
-      - name: Get branch names
-        id: branch-name
-        uses: tj-actions/branch-names@v6
-
       - name: Update version.py
         run: |
           export PYTHONPATH=$PYTHONPATH:flowcept
-          export BRANCH_NAME="${{ steps.branch-name.outputs.current_branch }}"
           python .github/workflows/version_bumper.py
 
       - name: Commit new version

.github/workflows/run-tests-in-container.yml

@@ -1,4 +1,4 @@
-name: Tests inside a Container
+name: (With Mongo) Tests inside a Container
 on: [pull_request]
 
 jobs:
@@ -20,9 +20,18 @@ jobs:
         run: make build
 
       - name: Start dependent services (Mongo and Redis)
-        run: make services
+        run: make services-mongo
 
       - name: Run tests in container
+        run: make tests-in-container-mongo
+
+      - name: Stop services
+        run: make services-stop-mongo
+
+      - name: Start dependent services (Redis, no Mongo)
+        run: make services
+
+      - name: Run tests in container without mongo
         run: make tests-in-container
 
       - name: Stop services

.github/workflows/run-tests-kafka.yml

@@ -1,13 +1,18 @@
-name: All tests on Kafka MQ
+name: (With Mongo) Tests on Kafka MQ
 on:
   pull_request:
     branches: [ "dev", "main" ]
     types: [opened, synchronize, reopened]
+#on:
+#  push:
 
 jobs:
 
   build:
     runs-on: ubuntu-latest
+    env:
+      MONGO_ENABLED: true
+      LMDB_ENABLED: false
     timeout-minutes: 40
     if: "!contains(github.event.head_commit.message, 'CI Bot')"
 
@@ -22,17 +27,22 @@ jobs:
           python-version: "3.10"
           cache: "pip"
 
-      - name: Install package and dependencies
-        run: |
-          python -m pip install --upgrade pip
-          python -m pip install .[all]
-          python -m pip install .[ml_dev]
-
       - name: Run docker compose
         run: docker compose -f deployment/compose-kafka.yml up -d
 
-      - name: Wait for one minute
-        run: sleep 60
+      - name: Upgrade pip
+        run: python -m pip install --upgrade pip
+
+      - name: Show Python version
+        run: python --version && pip --version
+
+      - name: Test examples
+        run: bash .github/workflows/run_examples.sh examples true # with mongo
+
+      - name: Install all dependencies
+        run: |
+          python -m pip install .[all]
+          python -m pip install .[ml_dev]
 
       - name: Check liveness
         run: |
@@ -45,7 +55,7 @@ jobs:
         run: |
           export MQ_TYPE=kafka
           export MQ_PORT=9092
-          pytest --ignore=tests/decorator_tests/ml_tests/llm_tests
+          make tests
 
       - name: Test notebooks
         run: pytest --ignore=notebooks/zambeze.ipynb --nbmake "notebooks/" --nbmake-timeout=600 --ignore=notebooks/dask_from_CLI.ipynb

.github/workflows/run-tests-py11-all-dbs.yml

@@ -0,0 +1,61 @@
+name: (With and Without Mongo) Tests on py11
+on: [pull_request]
+
+jobs:
+
+  build:
+    runs-on: ubuntu-latest
+    timeout-minutes: 40
+    env:
+      MONGO_ENABLED: true
+      LMDB_ENABLED: true
+
+    if: "!contains(github.event.head_commit.message, 'CI Bot')"
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 1
+
+      - name: Set up Python 3.11
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+          cache: "pip"
+
+      - name: Show OS Info
+        run: '[[ "$OSTYPE" == "linux-gnu"* ]] && { echo "OS Type: Linux"; (command -v lsb_release &> /dev/null && lsb_release -a) || cat /etc/os-release; uname -r; } || [[ "$OSTYPE" == "darwin"* ]] && { echo "OS Type: macOS"; sw_vers; uname -r; } || echo "Unsupported OS type: $OSTYPE"'
+
+      - name: Start docker compose with redis
+        run: make services-mongo
+
+      - name: Upgrade pip
+        run: python -m pip install --upgrade pip
+
+      - name: Show Python version
+        run: python --version && pip --version
+
+      - name: Test examples
+        run: bash .github/workflows/run_examples.sh examples true # with mongo
+
+      - name: Install all dependencies
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip install .[all]
+          python -m pip install .[ml_dev]
+
+      - name: List installed packages
+        run: pip list
+
+      - name: Test with pytest and redis
+        run: |
+          export MONGO_ENABLED=true
+          export LMDB_ENABLED=true
+          python -c "from flowcept.configs import MONGO_ENABLED, LMDB_ENABLED; print('MONGO?', MONGO_ENABLED); print('LMDB?', LMDB_ENABLED)"
+          make tests
+
+      - name: Test notebooks with pytest and redis
+        run: pytest --nbmake "notebooks/" --nbmake-timeout=600 --ignore="notebooks/dask_from_CLI.ipynb"
+
+      - name: Stop services
+        run: make services-stop-mongo

.github/workflows/run-tests-py11-simple.yml

@@ -0,0 +1,56 @@
+name: (Without Mongo) Tests on py11
+on: [pull_request]
+
+jobs:
+
+  build:
+    runs-on: ubuntu-latest
+    timeout-minutes: 40
+    env:
+      MONGO_ENABLED: false
+    if: "!contains(github.event.head_commit.message, 'CI Bot')"
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 1
+
+      - name: Set up Python 3.11
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+          cache: "pip"
+
+      - name: Show OS Info
+        run: '[[ "$OSTYPE" == "linux-gnu"* ]] && { echo "OS Type: Linux"; (command -v lsb_release &> /dev/null && lsb_release -a) || cat /etc/os-release; uname -r; } || [[ "$OSTYPE" == "darwin"* ]] && { echo "OS Type: macOS"; sw_vers; uname -r; } || echo "Unsupported OS type: $OSTYPE"'
+
+      - name: Start docker compose with redis
+        run: make services
+
+      - name: Upgrade pip
+        run: python -m pip install --upgrade pip
+
+      - name: Show Python version
+        run: python --version && pip --version
+
+      - name: Test examples
+        run: bash .github/workflows/run_examples.sh examples false # without mongo
+
+      - name: Install all dependencies
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip install .[all]
+          python -m pip install .[ml_dev]
+
+      - name: List installed packages
+        run: pip list
+
+      - name: Test with pytest and redis
+        run: |
+          make tests
+
+      - name: Test notebooks with pytest and redis
+        run: pytest --nbmake "notebooks/" --nbmake-timeout=600 --ignore="notebooks/dask_from_CLI.ipynb" --ignore="notebooks/analytics.ipynb"
+
+      - name: Stop services
+        run: make services-stop

.github/workflows/run-tests-py11.yml

@@ -1,11 +1,14 @@
-name: Tests on py11
+name: (With Mongo) Tests on py11
 on: [pull_request]
 
 jobs:
 
   build:
     runs-on: ubuntu-latest
     timeout-minutes: 40
+    env:
+      MONGO_ENABLED: true
+      LMDB_ENABLED: false
     if: "!contains(github.event.head_commit.message, 'CI Bot')"
 
     steps:
@@ -23,40 +26,19 @@ jobs:
         run: '[[ "$OSTYPE" == "linux-gnu"* ]] && { echo "OS Type: Linux"; (command -v lsb_release &> /dev/null && lsb_release -a) || cat /etc/os-release; uname -r; } || [[ "$OSTYPE" == "darwin"* ]] && { echo "OS Type: macOS"; sw_vers; uname -r; } || echo "Unsupported OS type: $OSTYPE"'
 
       - name: Start docker compose with redis
-        run: make services
+        run: make services-mongo
 
       - name: Upgrade pip
         run: python -m pip install --upgrade pip
 
       - name: Show Python version
         run: python --version && pip --version
 
-      - name: Install default dependencies and run simple test
-        run: |          
-          pip install .
-          python examples/simple_instrumented_script.py
-
-      - name: Install Dask dependencies alone and run a simple Dask test
-        run: |          
-          pip uninstall flowcept -y
-          pip install .[dask]
-          python examples/dask_example.py          
-
-      - name: Install MLFlow dependencies alone and run a simple MLFlow test
-        run: |          
-          pip uninstall flowcept -y
-          pip install .[mlflow]
-          python examples/mlflow_example.py    
-
-      - name: Install Tensorboard dependencies alone and run a simple Tensorboard test
-        run: |          
-          pip uninstall flowcept -y
-          pip install .[tensorboard]
-          python examples/tensorboard_example.py              
+      - name: Test examples
+        run: bash .github/workflows/run_examples.sh examples true # with mongo
 
       - name: Install all dependencies
         run: |
-          python -m pip install --upgrade pip
           python -m pip install .[all]
           python -m pip install .[ml_dev]
 
@@ -71,7 +53,7 @@ jobs:
         run: make tests-notebooks
 
       - name: Stop services
-        run: make services-stop
+        run: make services-stop-mongo
 
       - name: Clean up
         run: |

.github/workflows/run-tests-simple.yml

@@ -0,0 +1,61 @@
+name: (Without Mongo) Unit, integration, and notebook tests
+on:
+  push:
+  schedule:
+    - cron: '0 12 * * *'  # Runs every day at 12 PM UTC (7 AM EST)
+
+jobs:
+
+  build:
+    runs-on: ubuntu-latest
+    env:
+      MONGO_ENABLED: false
+      LMDB_ENABLED: true
+    timeout-minutes: 60
+    if: "!contains(github.event.head_commit.message, 'CI Bot')"
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 1
+
+      - name: Set up Python 3.10
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.10"
+          cache: "pip"
+
+      - name: Show OS Info
+        run: '[[ "$OSTYPE" == "linux-gnu"* ]] && { echo "OS Type: Linux"; (command -v lsb_release &> /dev/null && lsb_release -a) || cat /etc/os-release; uname -r; } || [[ "$OSTYPE" == "darwin"* ]] && { echo "OS Type: macOS"; sw_vers; uname -r; } || echo "Unsupported OS type: $OSTYPE"'
+
+      - name: Start docker compose with redis
+        run: make services
+
+      - name: Upgrade pip
+        run: python -m pip install --upgrade pip
+
+      - name: Test examples
+        run: bash .github/workflows/run_examples.sh examples false # with mongo
+
+      - name: Install all dependencies
+        run: |
+          python -m pip install .[all]
+          python -m pip install .[ml_dev]
+
+      - name: List installed packages
+        run: pip list
+
+      - name: Test with pytest and redis
+        run: |
+          make tests
+
+      - name: Test notebooks with pytest and redis
+        run: pytest --nbmake "notebooks/" --nbmake-timeout=600 --ignore="notebooks/dask_from_CLI.ipynb" --ignore="notebooks/analytics.ipynb"
+
+      - name: Shut down docker compose
+        run: make services-stop
+
+      - name: Clean up
+        run: |
+          make clean
+          find /home/runner/runners/ -type f -name "*.log" -exec sh -c 'echo {}; >"{}"' \;