run-tests.yml

name: (With Mongo) Unit, integration, and notebook tests
on:
  push:
  schedule:
    - cron: '0 12 * * *'  # Runs every day at 12 PM UTC (7 AM EST)

jobs:

  build:
    runs-on: ubuntu-latest
    env:
      MONGO_ENABLED: true
      LMDB_ENABLED: false
    timeout-minutes: 60
    if: "!contains(github.event.head_commit.message, 'CI Bot')"

    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 1

      - name: Set up Python 3.10
        uses: actions/setup-python@v5
        with:
          python-version: "3.10"
          cache: "pip"

      - name: Show OS Info
        run: '[[ "$OSTYPE" == "linux-gnu"* ]] && { echo "OS Type: Linux"; (command -v lsb_release &> /dev/null && lsb_release -a) || cat /etc/os-release; uname -r; } || [[ "$OSTYPE" == "darwin"* ]] && { echo "OS Type: macOS"; sw_vers; uname -r; } || echo "Unsupported OS type: $OSTYPE"'

      - name: Start docker compose with redis
        run: make services-mongo

      - name: Upgrade pip
        run: python -m pip install --upgrade pip

      - name: Install basic deps
        run: pip install .[mongo]

      - name: Check dbs utilization
        run: python -c "from flowcept.configs import MONGO_ENABLED, LMDB_ENABLED; print('MONGO?', MONGO_ENABLED); print('LMDB?', LMDB_ENABLED)"

      - name: Run simple test
        run: |
          python examples/simple_instrumented_script.py | tee output.log
          cat output.log
          grep -q "ERROR" output.log && exit 1
          rm output.log
 

      - name: Install Dask dependencies alone and run a simple Dask test
        run: |
          pip uninstall flowcept -y
          pip install .[dask]
          python examples/dask_example.py | tee output.log
          cat output.log
          grep -q "ERROR" output.log && exit 1
          rm output.log          

      - name: Install MLFlow dependencies alone and run a simple MLFlow test
        run: |
          pip uninstall flowcept -y
          pip install .[mlflow]
          python examples/mlflow_example.py | tee output.log
          cat output.log
          grep -q "ERROR" output.log && exit 1
          rm output.log    

      - name: Install Tensorboard dependencies alone and run a simple Tensorboard test
        run: |
          pip uninstall flowcept -y
          pip install .[tensorboard]
          python examples/tensorboard_example.py  | tee output.log
          cat output.log
          grep -q "ERROR" output.log && exit 1
          rm output.log 

      - name: Install all dependencies
        run: |
          python -m pip install --upgrade pip
          python -m pip install .[all]
          python -m pip install .[ml_dev]

      - name: List installed packages
        run: pip list

      - name: Test with pytest and redis
        run: |
          make tests

      - name: Test notebooks with pytest and redis
        run: make tests-notebooks

      - name: Shut down docker compose
        run: make services-stop-mongo

      - name: Clean up
        run: |
          make clean
          find /home/runner/runners/ -type f -name "*.log" -exec sh -c 'echo {}; >"{}"' \;

      - name: Start docker compose with kafka
        run: docker compose -f deployment/compose-kafka.yml up -d

      - name: Wait for one minute
        run: sleep 60

      - name: Check liveness
        run: |
          export MQ_TYPE=kafka
          export MQ_PORT=9092
          python -c 'from flowcept.configs import MQ_TYPE, MQ_PORT; print(f"MQ_TYPE={MQ_TYPE}"); print(f"MQ_PORT={MQ_PORT}")'
          python -c 'from flowcept import Flowcept; assert Flowcept.services_alive()'

      - name: Run tests with kafka
        run: |
          export MQ_TYPE=kafka
          export MQ_PORT=9092
          make tests

      - name: Stop docker compose with kafka
        run: docker compose -f deployment/compose-kafka.yml down

      - name: Clean up
        run: |
          make clean
          find /home/runner/runners/ -type f -name "*.log" -exec sh -c 'echo {}; >"{}"' \; || true
          docker image prune -a -f

      - name: List large files
        run: find . -type f -exec du -h {} + | sort -h