Merge pull request #156 from ORNL/dev

Main < Dev: setup.py -> toml

.github/workflows/create-release-n-publish.yml

@@ -2,21 +2,22 @@ name: Release and Publish
 on:
   push:  # It has to be push, otherwise error happens in code below.
     branches: [ "main" ]
-    #branches: [ "main", "dev" ] # use this only to test the CI
+    # branches: [ "main", "dev" ] # use this only to test the CI
     #branches: [ "disabled" ]
 jobs:
   build:
     name: Create Release and Publish
     runs-on: ubuntu-latest
-#    env:
-#      FLOWCEPT_SETTINGS_PATH: 'resources/settings.yaml'
+    timeout-minutes: 60
     steps:
       - name: Checkout code
-        uses: actions/checkout@v2
-      - name: Set up Python 3.9
-        uses: actions/setup-python@v3
+        uses: actions/checkout@v4
+
+      - name: Set up Python 3.10
+        uses: actions/setup-python@v5
         with:
-          python-version: "3.9"
+          python-version: "3.10"
+
       - name: Get branch names
         id: branch-name
         uses: tj-actions/branch-names@v6
@@ -30,12 +31,13 @@ jobs:
           git config --global user.name 'Flowcept CI Bot'
           git config --global user.email '[email protected]'
           git branch
-          git add flowcept/version.py
+          git add src/flowcept/version.py
           git commit -m "Flowcept CI Bot: bumping master version"
           git push --force
+
       - name: Get Latest PR and Create Release
         run: |
-            export CURRENT_VERSION=`python -c "f = open('flowcept/version.py'); exec(f.read()); print(locals()['__version__']); f.close()"`
+            export CURRENT_VERSION=`python -c "f = open('src/flowcept/version.py'); exec(f.read()); print(locals()['__version__']); f.close()"`
             echo $CURRENT_VERSION
             REPOSITORY=${{ github.repository }}
             ACCESS_TOKEN=${{ secrets.GITHUB_TOKEN  }}
@@ -61,6 +63,7 @@ jobs:
                             \"prerelease\": false}" \
                 -H "Authorization: Bearer ${ACCESS_TOKEN}" \
                 https://api.github.com/repos/${REPOSITORY}/releases
+
       - name: Install pypa/build
         run: >-
           python -m
@@ -81,6 +84,7 @@ jobs:
           password: ${{ secrets.TEST_PYPI_API_TOKEN }}
           repository_url: https://test.pypi.org/legacy/
           verbose: true
+
       - name: Publish distribution to PyPI
         #if: startsWith(github.ref, 'refs/tags')
         uses: pypa/gh-action-pypi-publish@release/v1
@@ -98,11 +102,11 @@ jobs:
       - name: Test pip install multiple adapters
         run: pip install flowcept[mlflow,tensorboard]
       - name: Install our dependencies
-        run: pip install flowcept[fulldev]  # This will install all dependencies, for all adapters and dev deps.
+        run: pip install flowcept[all]  # This will install all dependencies, for all adapters and dev deps.
       - name: Pip list
         run: pip list      
       - name: Run Docker Compose
-        run: docker compose -f deployment/compose-full.yml up -d
+        run: docker compose -f deployment/compose.yml up -d
       - name: Test with pytest
         run: |
           mkdir -p ~/.flowcept

.github/workflows/run-checks.yml

@@ -0,0 +1,30 @@
+name: Ruff linter and formatter checks
+on: [pull_request]
+
+permissions:
+  contents: read
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    if: "!contains(github.event.head_commit.message, 'CI Bot')"
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python 3.10
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.10"
+          cache: "pip"
+
+      - name: Install package and dependencies
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip install .[all]
+
+      - name: Run ruff linter checks
+        run: ruff check src
+
+      - name: Run ruff formatter checks
+        run: ruff format --check src

.github/workflows/run-tests-kafka.yml

@@ -3,52 +3,62 @@ on:
   pull_request:
     branches: [ "dev", "main" ]
     types: [opened, synchronize, reopened]
-# branches: [ "disabled" ]
 
 jobs:
 
   build:
     runs-on: ubuntu-latest
-    timeout-minutes: 60
+    timeout-minutes: 40
     if: "!contains(github.event.head_commit.message, 'CI Bot')"
+
     steps:
-    - uses: actions/checkout@v3
-    - name: Set up Python 3.9
-      uses: actions/setup-python@v3
-      with:
-        python-version: "3.9"
-    - name: Check python version
-      run: python --version
-    - name: Install our dependencies
-      run: |
-        python -m pip install --upgrade pip
-        pip install -e .[fulldev]
-    - name: Pip list
-      run: pip list
-    - name: Run Docker Compose
-      run: docker compose -f deployment/compose-kafka.yml up -d
-    - name: Wait 1 min
-      run: sleep 60
-    - name: Check liveness
-      run: |
-        export MQ_TYPE=kafka
-        export MQ_PORT=9092
-        python -c 'from flowcept.configs import MQ_TYPE, MQ_PORT; print(f"MQ_TYPE={MQ_TYPE}"); print(f"MQ_TYPE={MQ_PORT}")'
-        python -c 'from flowcept import Flowcept; assert Flowcept.services_alive()'
-    - name: Run Tests with Kafka
-      run: |
-        export MQ_TYPE=kafka
-        export MQ_PORT=9092
-        pytest --ignore=tests/decorator_tests/ml_tests/llm_tests
-    - name: Test notebooks
-      run: |
-        pip install -e .[full]
-        export MQ_TYPE=kafka
-        export MQ_PORT=9092
-        python -c 'from flowcept.configs import MQ_TYPE, MQ_PORT; print(f"MQ_TYPE={MQ_TYPE}"); print(f"MQ_TYPE={MQ_PORT}")'
-        python -c 'from flowcept import Flowcept; assert Flowcept.services_alive()'
-        
-        python flowcept/flowcept_webserver/app.py &
-        sleep 3
-        export FLOWCEPT_SETTINGS_PATH=~/.flowcept/settings.yaml
-        pytest --ignore=notebooks/zambeze.ipynb --nbmake "notebooks/" --nbmake-timeout=600 --ignore=notebooks/dask_from_CLI.ipynb
+      - uses: actions/checkout@v4
+
+      - name: Set up Python 3.10
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.10"
+          cache: "pip"
+
+      - name: Copy settings file
+        run: |
+          mkdir ~/.flowcept
+          cp resources/sample_settings.yaml ~/.flowcept
+          mv ~/.flowcept/sample_settings.yaml ~/.flowcept/settings.yaml
+
+      - name: Install package and dependencies
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip install .[all]
+
+      - name: Run docker compose
+        run: docker compose -f deployment/compose-kafka.yml up -d
+
+      - name: Wait for one minute
+        run: sleep 60
+
+      - name: Check liveness
+        run: |
+          export MQ_TYPE=kafka
+          export MQ_PORT=9092
+          python -c 'from flowcept.configs import MQ_TYPE, MQ_PORT; print(f"MQ_TYPE={MQ_TYPE}"); print(f"MQ_PORT={MQ_PORT}")'
+          python -c 'from flowcept import Flowcept; assert Flowcept.services_alive()'
+
+      - name: Run tests with kafka
+        run: |
+          export MQ_TYPE=kafka
+          export MQ_PORT=9092
+          pytest --ignore=tests/decorator_tests/ml_tests/llm_tests
+
+      - name: Test notebooks
+        run: |
+          pip install -e .[all]
+          export MQ_TYPE=kafka
+          export MQ_PORT=9092
+          python -c 'from flowcept.configs import MQ_TYPE, MQ_PORT; print(f"MQ_TYPE={MQ_TYPE}"); print(f"MQ_PORT={MQ_PORT}")'
+          python -c 'from flowcept import Flowcept; assert Flowcept.services_alive()'
+
+          python flowcept/flowcept_webserver/app.py &
+          sleep 3
+          export FLOWCEPT_SETTINGS_PATH=~/.flowcept/settings.yaml
+          pytest --ignore=notebooks/zambeze.ipynb --nbmake "notebooks/" --nbmake-timeout=600 --ignore=notebooks/dask_from_CLI.ipynb