From 856d3d1ec00e778cc1e5c1df3851337a761c8f32 Mon Sep 17 00:00:00 2001 From: Tanja Bunk Date: Fri, 20 Dec 2024 09:40:12 +0100 Subject: [PATCH] undo changes to CI workflow --- .github/workflows/continous-integration.yml | 126 ++++++++++++++++---- 1 file changed, 103 insertions(+), 23 deletions(-) diff --git a/.github/workflows/continous-integration.yml b/.github/workflows/continous-integration.yml index d29b52b2ee5d..00f2f8aab5f8 100644 --- a/.github/workflows/continous-integration.yml +++ b/.github/workflows/continous-integration.yml @@ -290,6 +290,7 @@ jobs: - name: Prevent race condition in poetry build # More context about race condition during poetry build can be found here: # https://github.com/python-poetry/poetry/issues/7611#issuecomment-1747836233 + if: needs.changes.outputs.backend == 'true' run: | poetry config installer.max-workers 1 @@ -395,7 +396,7 @@ jobs: - name: Store coverage reports if: needs.changes.outputs.backend == 'true' && matrix.os == 'ubuntu-22.04' - uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce + uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 #v4.4.3 with: name: ${{ matrix.test }}-coverage path: | @@ -546,7 +547,7 @@ jobs: - name: Store coverage reports if: needs.changes.outputs.backend == 'true' && matrix.os == 'ubuntu-22.04' - uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce + uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 #v4.4.3 with: name: ${{ matrix.test }}-coverage path: | @@ -573,7 +574,7 @@ jobs: - name: Get backend coverage reports if: needs.changes.outputs.backend == 'true' - uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a + uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 #v4.1.8 with: path: ${{ github.workspace }}/tests_coverage @@ -617,10 +618,6 @@ jobs: POSTGRES_PORT: 5432 POSTGRES_USER: postgres POSTGRES_PASSWORD: postgres - RABBITMQ_HOST: localhost - RABBITMQ_PORT: 5672 - RABBITMQ_USER: guest - RABBITMQ_PASSWORD: guest services: redis: @@ -652,12 +649,6 @@ jobs: # mapping container ports to the host - 5432:5432 - rabbitmq: - # see https://github.com/docker-library/healthcheck/blob/master/rabbitmq/docker-healthcheck - image: healthcheck/rabbitmq - ports: - - 5672:5672 - mongodb: image: mongodb/mongodb-community-server:6.0.4-ubuntu2204 options: >- @@ -727,6 +718,94 @@ jobs: if grep 'The lock file is not up to date' .output; then exit 1; fi make prepare-tests-ubuntu + - name: Test Code with Services 🩺 + if: needs.changes.outputs.backend == 'true' + env: + JOBS: 2 + INTEGRATION_TEST_PYTEST_MARKERS: '"(not sequential) and (not broker)"' + PYTHONIOENCODING: "utf-8" + run: | + make test-integration + + broker_integration_test: + name: Run Broker Integration Tests + if: github.ref_type != 'tag' + runs-on: ubuntu-22.04 + timeout-minutes: 60 + needs: [changes] + env: + RABBITMQ_HOST: localhost + RABBITMQ_PORT: 5672 + RABBITMQ_USER: guest + RABBITMQ_PASSWORD: guest + + services: + rabbitmq: + # see https://github.com/docker-library/healthcheck/blob/master/rabbitmq/docker-healthcheck + image: healthcheck/rabbitmq + ports: + - 5672:5672 + + steps: + - name: Checkout git repository 🕝 + if: needs.changes.outputs.backend == 'true' + uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c + + - name: Set up Python ${{ env.DEFAULT_PYTHON_VERSION }} 🐍 + if: needs.changes.outputs.backend == 'true' + uses: actions/setup-python@57ded4d7d5e986d7296eab16560982c6dd7c923b + with: + python-version: ${{ env.DEFAULT_PYTHON_VERSION }} + + - name: Read Poetry Version 🔢 + if: needs.changes.outputs.backend == 'true' + run: | + echo "POETRY_VERSION=$(scripts/poetry-version.sh)" >> $GITHUB_ENV + shell: bash + + - name: Install poetry 🦄 + if: needs.changes.outputs.backend == 'true' + uses: Gr1N/setup-poetry@15821dc8a61bc630db542ae4baf6a7c19a994844 # v8 + with: + poetry-version: ${{ env.POETRY_VERSION }} + + - name: Load Poetry Cached Libraries ⬇ + id: cache-poetry + if: needs.changes.outputs.backend == 'true' + uses: actions/cache@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8 + with: + path: .venv + key: ${{ runner.os }}-poetry-${{ env.POETRY_VERSION }}-${{ env.DEFAULT_PYTHON_VERSION }}-${{ hashFiles('**/poetry.lock') }}-venv-${{ secrets.POETRY_CACHE_VERSION }}-${{ env.pythonLocation }} + + - name: Clear Poetry cache + if: steps.cache-poetry.outputs.cache-hit == 'true' && needs.changes.outputs.backend == 'true' && contains(github.event.pull_request.labels.*.name, 'tools:clear-poetry-cache-unit-tests') + run: rm -r .venv + + # Poetry >= 1.1.0b uses virtualenv to create a virtual environment. + # The virtualenv simply doesn't work on Windows with our setup, + # that's why we use venv to create virtual environment + - name: Create virtual environment + if: (steps.cache-poetry.outputs.cache-hit != 'true' || contains(github.event.pull_request.labels.*.name, 'tools:clear-poetry-cache-unit-tests')) && needs.changes.outputs.backend == 'true' + run: python -m venv create .venv + + - name: Set up virtual environment + if: needs.changes.outputs.backend == 'true' + # Poetry on Windows cannot pick up the virtual environments directory properly, + # and it creates a new one every time the pipeline runs. + # This step solves this problem — it tells poetry to always use `.venv` directory inside + # the project itself, which also makes it easier for us to determine the correct directory + # that needs to be cached. + run: poetry config virtualenvs.in-project true + + - name: Install Dependencies (Linux) 📦 + if: needs.changes.outputs.backend == 'true' + run: | + sudo apt-get -y install libpq-dev + make install-full | tee .output + if grep 'The lock file is not up to date' .output; then exit 1; fi + make prepare-tests-ubuntu + make prepare-spacy + - name: Run kafka and zookeeper containers for integration testing if: needs.changes.outputs.backend == 'true' run: | @@ -736,11 +815,16 @@ jobs: if: needs.changes.outputs.backend == 'true' env: JOBS: 2 - INTEGRATION_TEST_PYTEST_MARKERS: '"not sequential"' + INTEGRATION_TEST_PYTEST_MARKERS: "broker" PYTHONIOENCODING: "utf-8" run: | make test-integration + - name: Stop kafka and zookeeper containers for integration testing + if: needs.changes.outputs.backend == 'true' + run: | + docker-compose -f tests_deployment/docker-compose.kafka.yml down + sequential_integration_test: name: Run Sequential Integration Tests if: github.ref_type != 'tag' @@ -840,11 +924,6 @@ jobs: run: | make test-integration - - name: Stop kafka and zookeeper containers for integration testing - if: needs.changes.outputs.backend == 'true' - run: | - docker-compose -f tests_deployment/docker-compose.kafka.yml down - build_docker_base_images_and_set_env: name: Build Docker base images and setup environment runs-on: ubuntu-22.04 @@ -1196,14 +1275,15 @@ jobs: pip install -U github3.py pep440-version-utils python3 scripts/publish_gh_release_notes.py - - name: Notify Slack of successful release 💬 + - name: Notify Slack of successful release # notification will be sent to the #product channel on slack, webhook url is added as repository secret if: success() env: - SLACK_WEBHOOK: ${{ secrets.SLACK_RELEASE_ASSISTANT_PRODUCT_WEBHOOK }} + SLACK_WEBHOOK: ${{ secrets.SLACK_RELEASE_ASSISTANT_RELEASE_WEBHOOK }} uses: Ilshidur/action-slack@689ad44a9c9092315abd286d0e3a9a74d31ab78a with: - args: "💥 New *Rasa Private* version `${{ github.ref_name }}` has been released!" + args: "💥 New *Rasa Open Source * version `${{ github.ref_name }}` has been released!" + send_slack_notification_for_release_on_failure: name: Notify Slack & Publish Release Notes @@ -1218,7 +1298,7 @@ jobs: # send notification if 'deploy' is skipped (previous needed job failed) or failed if: needs.deploy.result != 'success' env: - SLACK_WEBHOOK: ${{ secrets.SLACK_RELEASE_ASSISTANT_ATOM_ALERTS_WEBHOOK }} + SLACK_WEBHOOK: ${{ secrets.SLACK_RELEASE_ASSISTANT_DEV_TRIBE_WEBHOOK }} uses: Ilshidur/action-slack@689ad44a9c9092315abd286d0e3a9a74d31ab78a with: args: "⛔️ *Rasa Open Source* version `${{ github.ref_name }}` could not be released 😱! Please check out GitHub Actions: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}"