Merge branch '3.6.x' into pickle-alternatives-3.6.x

.github/poetry_version.txt

@@ -1,2 +1,2 @@
 # The poetry version is stored in a separate file due to the https://github.com/python-poetry/poetry/issues/3316
-poetry-version=1.4.2
+poetry-version=1.8.2

.github/workflows/continous-integration.yml

@@ -290,6 +290,7 @@ jobs:
       - name: Prevent race condition in poetry build
         # More context about race condition during poetry build can be found here:
         # https://github.com/python-poetry/poetry/issues/7611#issuecomment-1747836233
+        if: needs.changes.outputs.backend == 'true'
         run: |
           poetry config installer.max-workers 1
 
@@ -395,7 +396,7 @@ jobs:
 
       - name: Store coverage reports
         if: needs.changes.outputs.backend == 'true' && matrix.os == 'ubuntu-22.04'
-        uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce
+        uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 #v4.4.3
         with:
           name: ${{ matrix.test }}-coverage
           path: |
@@ -546,7 +547,7 @@ jobs:
 
       - name: Store coverage reports
         if: needs.changes.outputs.backend == 'true' && matrix.os == 'ubuntu-22.04'
-        uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce
+        uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 #v4.4.3
         with:
           name: ${{ matrix.test }}-coverage
           path: |
@@ -573,7 +574,7 @@ jobs:
 
       - name: Get backend coverage reports
         if: needs.changes.outputs.backend == 'true'
-        uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a
+        uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 #v4.1.8
         with:
           path: ${{ github.workspace }}/tests_coverage
 
@@ -617,10 +618,6 @@ jobs:
       POSTGRES_PORT: 5432
       POSTGRES_USER: postgres
       POSTGRES_PASSWORD: postgres
-      RABBITMQ_HOST: localhost
-      RABBITMQ_PORT: 5672
-      RABBITMQ_USER: guest
-      RABBITMQ_PASSWORD: guest
 
     services:
       redis:
@@ -652,12 +649,6 @@ jobs:
           # mapping container ports to the host
           - 5432:5432
 
-      rabbitmq:
-        # see https://github.com/docker-library/healthcheck/blob/master/rabbitmq/docker-healthcheck
-        image: healthcheck/rabbitmq
-        ports:
-          - 5672:5672
-
       mongodb:
         image: mongodb/mongodb-community-server:6.0.4-ubuntu2204
         options: >-
@@ -727,6 +718,94 @@ jobs:
           if grep 'The lock file is not up to date' .output; then exit 1; fi
           make prepare-tests-ubuntu
 
+      - name: Test Code with Services 🩺
+        if: needs.changes.outputs.backend == 'true'
+        env:
+          JOBS: 2
+          INTEGRATION_TEST_PYTEST_MARKERS: '"(not sequential) and (not broker)"'
+          PYTHONIOENCODING: "utf-8"
+        run: |
+          make test-integration
+
+  broker_integration_test:
+    name: Run Broker Integration Tests
+    if: github.ref_type != 'tag'
+    runs-on: ubuntu-22.04
+    timeout-minutes: 60
+    needs: [changes]
+    env:
+      RABBITMQ_HOST: localhost
+      RABBITMQ_PORT: 5672
+      RABBITMQ_USER: guest
+      RABBITMQ_PASSWORD: guest
+
+    services:
+      rabbitmq:
+        # see https://github.com/docker-library/healthcheck/blob/master/rabbitmq/docker-healthcheck
+        image: healthcheck/rabbitmq
+        ports:
+          - 5672:5672
+
+    steps:
+      - name: Checkout git repository 🕝
+        if: needs.changes.outputs.backend == 'true'
+        uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c
+
+      - name: Set up Python ${{ env.DEFAULT_PYTHON_VERSION }} 🐍
+        if: needs.changes.outputs.backend == 'true'
+        uses: actions/setup-python@57ded4d7d5e986d7296eab16560982c6dd7c923b
+        with:
+          python-version: ${{ env.DEFAULT_PYTHON_VERSION }}
+
+      - name: Read Poetry Version 🔢
+        if: needs.changes.outputs.backend == 'true'
+        run: |
+          echo "POETRY_VERSION=$(scripts/poetry-version.sh)" >> $GITHUB_ENV
+        shell: bash
+
+      - name: Install poetry 🦄
+        if: needs.changes.outputs.backend == 'true'
+        uses: Gr1N/setup-poetry@15821dc8a61bc630db542ae4baf6a7c19a994844 # v8
+        with:
+          poetry-version: ${{ env.POETRY_VERSION }}
+
+      - name: Load Poetry Cached Libraries ⬇
+        id: cache-poetry
+        if: needs.changes.outputs.backend == 'true'
+        uses: actions/cache@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8
+        with:
+          path: .venv
+          key: ${{ runner.os }}-poetry-${{ env.POETRY_VERSION }}-${{ env.DEFAULT_PYTHON_VERSION }}-${{ hashFiles('**/poetry.lock') }}-venv-${{ secrets.POETRY_CACHE_VERSION }}-${{ env.pythonLocation }}
+
+      - name: Clear Poetry cache
+        if: steps.cache-poetry.outputs.cache-hit == 'true' && needs.changes.outputs.backend == 'true' && contains(github.event.pull_request.labels.*.name, 'tools:clear-poetry-cache-unit-tests')
+        run: rm -r .venv
+
+        # Poetry >= 1.1.0b uses virtualenv to create a virtual environment.
+        # The virtualenv simply doesn't work on Windows with our setup,
+        # that's why we use venv to create virtual environment
+      - name: Create virtual environment
+        if: (steps.cache-poetry.outputs.cache-hit != 'true' || contains(github.event.pull_request.labels.*.name, 'tools:clear-poetry-cache-unit-tests')) && needs.changes.outputs.backend == 'true'
+        run: python -m venv create .venv
+
+      - name: Set up virtual environment
+        if: needs.changes.outputs.backend == 'true'
+        # Poetry on Windows cannot pick up the virtual environments directory properly,
+        # and it creates a new one every time the pipeline runs.
+        # This step solves this problem — it tells poetry to always use `.venv` directory inside
+        # the project itself, which also makes it easier for us to determine the correct directory
+        # that needs to be cached.
+        run: poetry config virtualenvs.in-project true
+
+      - name: Install Dependencies (Linux) 📦
+        if: needs.changes.outputs.backend == 'true'
+        run: |
+          sudo apt-get -y install libpq-dev
+          make install-full | tee .output
+          if grep 'The lock file is not up to date' .output; then exit 1; fi
+          make prepare-tests-ubuntu
+          make prepare-spacy
+
       - name: Run kafka and zookeeper containers for integration testing
         if: needs.changes.outputs.backend == 'true'
         run: |
@@ -736,11 +815,16 @@ jobs:
         if: needs.changes.outputs.backend == 'true'
         env:
           JOBS: 2
-          INTEGRATION_TEST_PYTEST_MARKERS: '"not sequential"'
+          INTEGRATION_TEST_PYTEST_MARKERS: "broker"
           PYTHONIOENCODING: "utf-8"
         run: |
           make test-integration
 
+      - name: Stop kafka and zookeeper containers for integration testing
+        if: needs.changes.outputs.backend == 'true'
+        run: |
+          docker-compose -f tests_deployment/docker-compose.kafka.yml down
+
   sequential_integration_test:
     name: Run Sequential Integration Tests
     if: github.ref_type != 'tag'
@@ -840,11 +924,6 @@ jobs:
         run: |
           make test-integration
 
-      - name: Stop kafka and zookeeper containers for integration testing
-        if: needs.changes.outputs.backend == 'true'
-        run: |
-          docker-compose -f tests_deployment/docker-compose.kafka.yml down
-
   build_docker_base_images_and_set_env:
     name: Build Docker base images and setup environment
     runs-on: ubuntu-22.04
@@ -1196,14 +1275,14 @@ jobs:
           pip install -U github3.py pep440-version-utils
           python3 scripts/publish_gh_release_notes.py
 
-      - name: Notify Slack of successful release 💬
+      - name: Notify Slack of successful release
         # notification will be sent to the #product channel on slack, webhook url is added as repository secret
         if: success()
         env:
-          SLACK_WEBHOOK: ${{ secrets.SLACK_RELEASE_ASSISTANT_PRODUCT_WEBHOOK }}
+          SLACK_WEBHOOK: ${{ secrets.SLACK_RELEASE_ASSISTANT_RELEASE_WEBHOOK }}
         uses: Ilshidur/action-slack@689ad44a9c9092315abd286d0e3a9a74d31ab78a
         with:
-          args: "💥 New *Rasa Private* version `${{ github.ref_name }}` has been released!"
+          args: "💥 New *Rasa Open Source * version `${{ github.ref_name }}` has been released!"
 
   send_slack_notification_for_release_on_failure:
     name: Notify Slack & Publish Release Notes
@@ -1218,7 +1297,7 @@ jobs:
         # send notification if 'deploy' is skipped (previous needed job failed) or failed
         if: needs.deploy.result != 'success'
         env:
-          SLACK_WEBHOOK: ${{ secrets.SLACK_RELEASE_ASSISTANT_ATOM_ALERTS_WEBHOOK }}
+          SLACK_WEBHOOK: ${{ secrets.SLACK_RELEASE_ASSISTANT_DEV_TRIBE_WEBHOOK }}
         uses: Ilshidur/action-slack@689ad44a9c9092315abd286d0e3a9a74d31ab78a
         with:
           args: "⛔️ *Rasa Open Source* version `${{ github.ref_name }}` could not be released 😱! Please check out GitHub Actions: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}"

.github/workflows/documentation.yml

@@ -82,7 +82,7 @@ jobs:
             echo "build_docs=true" >> $GITHUB_OUTPUT
           else
             # Get latest tagged Rasa version
-            git fetch --depth=1 origin "+refs/tags/*:refs/tags/*"
+            git describe --tags --match="3.6.[0-9]*" --abbrev=0 HEAD
             # Fetch branch history
             TAG_NAME=${GITHUB_REF#refs/tags/}
             git fetch --prune --unshallow

.github/workflows/security-scans.yml

@@ -129,24 +129,3 @@ jobs:
       - name: Run Bandit 🔪
         if: needs.changes.outputs.backend == 'true'
         run: make lint-security
-
-  snyk:
-    runs-on: ubuntu-22.04
-    steps:
-      - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c
-      - name: Run Snyk Open Source to check for Python vulnerabilities
-        uses: snyk/actions/python-3.8@master
-        continue-on-error: true
-        env:
-          SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }}
-        with:
-          command: monitor
-          args: --all-projects --org=rasa --skip-unresolved
-      - name: Run Snyk Open Source to check for JS vulnerabilities
-        uses: snyk/actions/node@master
-        continue-on-error: true
-        env:
-          SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }}
-        with:
-          command: monitor
-          args: --org=rasa --yarn-workspaces --strict-out-of-sync=false --prune-repeated-subdependencies