License only telemetry 3.4 #29373
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # The docs: | |
| # - https://www.notion.so/rasa/The-CI-for-model-regression-tests-aa579d5524a544af992f97d132bcc2de | |
| # - https://www.notion.so/rasa/Datadog-Usage-Documentation-422099c9a3a24f5a99d92d904537dd0b | |
| name: CI - Model Regression | |
| on: | |
| push: | |
| branches: | |
| - "[0-9]+.[0-9]+.x" | |
| tags: | |
| - "**" | |
| pull_request: | |
| types: [opened, synchronize, labeled] | |
| concurrency: | |
| group: ci-model-regression-${{ github.ref }} # branch or tag name | |
| cancel-in-progress: true | |
| env: | |
| GKE_ZONE: us-central1 | |
| GCLOUD_VERSION: "318.0.0" | |
| DD_PROFILING_ENABLED: false | |
| TF_FORCE_GPU_ALLOW_GROWTH: true | |
| NVML_INTERVAL_IN_SEC: 1 | |
| jobs: | |
| read_test_configuration: | |
| name: Reads tests configuration | |
| if: ${{ github.repository == 'RasaHQ/rasa' && contains(github.event.pull_request.labels.*.name, 'status:model-regression-tests') }} | |
| runs-on: ubuntu-22.04 | |
| outputs: | |
| matrix: ${{ steps.set-matrix.outputs.matrix }} | |
| matrix_length: ${{ steps.set-matrix.outputs.matrix_length }} | |
| configuration_id: ${{ steps.fc_config.outputs.comment-id }} | |
| dataset_branch: ${{ steps.set-dataset-branch.outputs.dataset_branch }} | |
| steps: | |
| - name: Checkout main | |
| uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c | |
| - name: Download gomplate | |
| run: |- | |
| sudo curl -o /usr/local/bin/gomplate -sSL https://github.com/hairyhenderson/gomplate/releases/download/v3.9.0/gomplate_linux-amd64 | |
| sudo chmod +x /usr/local/bin/gomplate | |
| - name: Find a comment with configuration | |
| uses: tczekajlo/find-comment@16228d0f2100e06ea9bf8c0e7fe7287b7c6b531d | |
| id: fc_config | |
| with: | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| issue-number: ${{ github.event.number }} | |
| body-includes: "^/modeltest" | |
| - run: echo ${{ steps.fc_config.outputs.comment-id }} | |
| # This step has to happen before the other configuration details are read from | |
| # the same PR comment, because we need to check out the correct branch to feed the | |
| # dataset mapping and configs into the 'Read configuration from a PR comment' step | |
| # which creates the experiments matrix | |
| - name: Read dataset branch from a PR comment | |
| if: steps.fc_config.outputs.comment-id != '' | |
| id: set-dataset-branch | |
| run: |- | |
| source <(gomplate -d github=https://api.github.com/repos/${{ github.repository }}/issues/comments/${{ steps.fc_config.outputs.comment-id }} -H 'github=Authorization:token ${{ secrets.GITHUB_TOKEN }}' -f .github/templates/model_regression_test_read_dataset_branch.tmpl) | |
| echo "dataset_branch=${DATASET_BRANCH}" >> $GITHUB_OUTPUT | |
| - name: Checkout dataset | |
| uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c | |
| with: | |
| repository: ${{ secrets.DATASET_REPOSITORY }} | |
| token: ${{ secrets.ML_TEST_SA_PAT }} | |
| path: "dataset" | |
| ref: ${{ steps.set-dataset-branch.outputs.dataset_branch }} | |
| - name: Render help description from template | |
| id: get_help_description | |
| run: | | |
| OUTPUT=$(gomplate -d mapping=./dataset/dataset_config_mapping.json -f .github/templates/model_regression_test_config_comment.tmpl) | |
| OUTPUT="${OUTPUT//$'\n'/'%0A'}" | |
| OUTPUT="${OUTPUT//$'\r'/'%0D'}" | |
| echo "help_description=$OUTPUT" >> $GITHUB_OUTPUT | |
| - name: Create a comment with help description | |
| uses: RasaHQ/create-comment@92ba126e8d110ef087528bf8d181c6e95827c95e | |
| with: | |
| mode: "delete-previous" | |
| id: comment_help_description | |
| github-token: ${{ secrets.GITHUB_TOKEN }} | |
| body: | | |
| ${{ steps.get_help_description.outputs.help_description }} | |
| - if: steps.fc_config.outputs.comment-id == '' | |
| run: echo "::error::Cannot find a comment with the configuration" | |
| name: Log a warning message if a configuration cannot be found | |
| - name: Read configuration from a PR comment | |
| if: steps.fc_config.outputs.comment-id != '' | |
| id: set-matrix | |
| run: |- | |
| matrix=$(gomplate -d mapping=./dataset/dataset_config_mapping.json -d github=https://api.github.com/repos/${{ github.repository }}/issues/comments/${{ steps.fc_config.outputs.comment-id }} -H 'github=Authorization:token ${{ secrets.GITHUB_TOKEN }}' -f .github/templates/model_regression_test_config_to_json.tmpl) | |
| if [ $? -ne 0 ]; then | |
| echo "::error::Cannot read config from PR. Please double check your config." | |
| exit 1 | |
| fi | |
| matrix_length=$(echo $matrix | jq '.[] | length') | |
| echo "matrix_length=$matrix_length" >> $GITHUB_OUTPUT | |
| echo "matrix=$matrix" >> $GITHUB_OUTPUT | |
| - name: Update the comment with the configuration | |
| uses: peter-evans/create-or-update-comment@67dcc547d311b736a8e6c5c236542148a47adc3d | |
| if: steps.fc_config.outputs.comment-id != '' | |
| with: | |
| comment-id: ${{ steps.fc_config.outputs.comment-id }} | |
| body: | | |
| <!-- comment-id:comment_configuration --> | |
| reactions: eyes | |
| - name: Re-create the comment with the configuration | |
| uses: RasaHQ/create-comment@92ba126e8d110ef087528bf8d181c6e95827c95e | |
| if: steps.fc_config.outputs.comment-id != '' && steps.fc_config.outputs.comment-body != '' | |
| with: | |
| mode: "delete-previous" | |
| id: comment_configuration | |
| github-token: ${{ secrets.GITHUB_TOKEN }} | |
| body: ${{ steps.fc_config.outputs.comment-body }} | |
| - name: Find a comment with configuration - update | |
| uses: tczekajlo/find-comment@16228d0f2100e06ea9bf8c0e7fe7287b7c6b531d | |
| id: fc_config_update | |
| with: | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| issue-number: ${{ github.event.number }} | |
| body-includes: "^/modeltest" | |
| - name: Add reaction | |
| uses: peter-evans/create-or-update-comment@67dcc547d311b736a8e6c5c236542148a47adc3d | |
| if: steps.fc_config_update.outputs.comment-id != '' | |
| with: | |
| edit-mode: "replace" | |
| comment-id: ${{ steps.fc_config_update.outputs.comment-id }} | |
| reactions: heart, hooray, rocket | |
| - name: Add a comment that the tests are in progress | |
| uses: RasaHQ/create-comment@92ba126e8d110ef087528bf8d181c6e95827c95e | |
| if: steps.fc_config_update.outputs.comment-id != '' | |
| with: | |
| mode: "delete-previous" | |
| id: comment_tests_in_progress | |
| github-token: ${{ secrets.GITHUB_TOKEN }} | |
| body: | | |
| The model regression tests have started. It might take a while, please be patient. | |
| As soon as results are ready you'll see a new comment with the results. | |
| Used configuration can be found in [the comment.](https://github.com/${{ github.repository }}/pull/${{ github.event.number}}#issuecomment-${{ steps.fc_config_update.outputs.comment-id }}) | |
| deploy_runner_gpu: | |
| name: Deploy Github Runner - GPU | |
| needs: read_test_configuration | |
| runs-on: ubuntu-22.04 | |
| if: ${{ contains(github.event.pull_request.labels.*.name, 'runner:gpu') && github.repository == 'RasaHQ/rasa' && contains(github.event.pull_request.labels.*.name, 'status:model-regression-tests') && needs.read_test_configuration.outputs.configuration_id != '' }} | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c | |
| - name: Download gomplate | |
| run: |- | |
| sudo curl -o /usr/local/bin/gomplate -sSL https://github.com/hairyhenderson/gomplate/releases/download/v3.9.0/gomplate_linux-amd64 | |
| sudo chmod +x /usr/local/bin/gomplate | |
| - name: Get TensorFlow version | |
| run: |- | |
| # Read TF version from poetry.lock file | |
| pip install toml | |
| TF_VERSION=$(scripts/read_tensorflow_version.sh) | |
| # Keep the first 3 characters, e.g. we keep 2.3 if TF_VERSION is 2.3.4 | |
| TF_VERSION=${TF_VERSION::3} | |
| echo "TensorFlow version: $TF_VERSION" | |
| echo TF_VERSION=$TF_VERSION >> $GITHUB_ENV | |
| # Use compatible CUDA/cuDNN with the given TF version | |
| - name: Prepare GitHub runner image tag | |
| run: |- | |
| GH_RUNNER_IMAGE_TAG=$(jq -r 'if (.config | any(.TF == "${{ env.TF_VERSION }}" )) then (.config[] | select(.TF == "${{ env.TF_VERSION }}") | .IMAGE_TAG) else .default_image_tag end' .github/configs/tf-cuda.json) | |
| echo "GitHub runner image tag for TensorFlow ${{ env.TF_VERSION }} is ${GH_RUNNER_IMAGE_TAG}" | |
| echo GH_RUNNER_IMAGE_TAG=$GH_RUNNER_IMAGE_TAG >> $GITHUB_ENV | |
| num_max_replicas=3 | |
| matrix_length=${{ needs.read_test_configuration.outputs.matrix_length }} | |
| if [[ $matrix_length -gt $num_max_replicas ]]; then | |
| NUM_REPLICAS=$num_max_replicas | |
| else | |
| NUM_REPLICAS=$matrix_length | |
| fi | |
| echo NUM_REPLICAS=$NUM_REPLICAS >> $GITHUB_ENV | |
| - name: Send warning if the current TF version does not have CUDA image tags configured | |
| if: env.GH_RUNNER_IMAGE_TAG == 'latest' | |
| env: | |
| TF_CUDA_FILE: ./github/config/tf-cuda.json | |
| run: |- | |
| echo "::warning file=${TF_CUDA_FILE},line=3,col=1,endColumn=3::Missing cuda config for tf ${{ env.TF_VERSION }}. If you are not sure how to config CUDA, please reach out to infrastructure." | |
| - name: Notify slack on tf-cuda config updates | |
| if: env.GH_RUNNER_IMAGE_TAG == 'latest' | |
| env: | |
| SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }} | |
| uses: voxmedia/github-action-slack-notify-build@212e9f7a9ca33368c8dd879d6053972128258985 | |
| with: | |
| channel_id: ${{ secrets.SLACK_ALERTS_CHANNEL_ID }} | |
| status: WARNING | |
| color: warning | |
| - name: Render deployment template | |
| run: |- | |
| export GH_RUNNER_IMAGE_TAG=${{ env.GH_RUNNER_IMAGE_TAG }} | |
| export GH_RUNNER_IMAGE=${{ secrets.GH_RUNNER_IMAGE }} | |
| gomplate -f .github/runner/github-runner-deployment.yaml.tmpl -o runner_deployment.yaml | |
| # Setup gcloud CLI | |
| - uses: google-github-actions/setup-gcloud@877d4953d2c70a0ba7ef3290ae968eb24af233bb | |
| with: | |
| version: "${{ env.GCLOUD_VERSION }}" | |
| service_account_key: ${{ secrets.GKE_SA_RASA_CI_CD_GPU_RASA_CI_CD }} | |
| service_account_email: ${{ secrets.GKE_RASA_CI_GPU_SA_NAME_RASA_CI_CD }} | |
| # Get the GKE credentials so we can deploy to the cluster | |
| - run: |- | |
| gcloud container clusters get-credentials "${{ secrets.GKE_GPU_CLUSTER_RASA_CI_CD }}" --zone "$GKE_ZONE" --project "${{ secrets.GKE_SA_RASA_CI_GPU_PROJECT_RASA_CI_CD }}" | |
| - name: Deploy Github Runner | |
| run: |- | |
| kubectl apply -f runner_deployment.yaml | |
| kubectl -n github-runner rollout status --timeout=15m deployment/github-runner-$GITHUB_RUN_ID | |
| model_regression_test_gpu: | |
| name: Model Regression Tests - GPU | |
| needs: | |
| - deploy_runner_gpu | |
| - read_test_configuration | |
| env: | |
| # Determine where CUDA and Nvidia libraries are located. TensorFlow looks for libraries in the given paths | |
| LD_LIBRARY_PATH: "/usr/local/cuda/extras/CUPTI/lib64:/usr/local/cuda/lib64:/usr/local/nvidia/lib:/usr/local/nvidia/lib64" | |
| ACCELERATOR_TYPE: "GPU" | |
| runs-on: [self-hosted, gpu, "${{ github.run_id }}"] | |
| strategy: | |
| # max-parallel: By default, GitHub will maximize the number of jobs run in parallel depending on the available runners on GitHub-hosted virtual machines. | |
| matrix: ${{fromJson(needs.read_test_configuration.outputs.matrix)}} | |
| fail-fast: false | |
| if: ${{ contains(github.event.pull_request.labels.*.name, 'runner:gpu') && github.repository == 'RasaHQ/rasa' && contains(github.event.pull_request.labels.*.name, 'status:model-regression-tests') && needs.read_test_configuration.outputs.configuration_id != '' }} | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c | |
| - name: Checkout dataset | |
| uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c | |
| with: | |
| repository: ${{ secrets.DATASET_REPOSITORY }} | |
| token: ${{ secrets.ML_TEST_SA_PAT }} | |
| path: "dataset" | |
| ref: ${{ needs.read_test_configuration.outputs.dataset_branch }} | |
| - name: Set env variables | |
| id: set_dataset_config_vars | |
| env: | |
| DATASET_NAME: "${{ matrix.dataset }}" | |
| CONFIG_NAME: "${{ matrix.config }}" | |
| run: |- | |
| # determine extra environment variables | |
| # - CONFIG | |
| # - DATASET | |
| # - IS_EXTERNAL | |
| # - EXTERNAL_DATASET_REPOSITORY_BRANCH | |
| # - TRAIN_DIR | |
| # - TEST_DIR | |
| # - DOMAIN_FILE | |
| source <(gomplate -d mapping=./dataset/dataset_config_mapping.json -f .github/templates/configuration_variables.tmpl) | |
| # Not all configurations are available for all datasets. | |
| # The job will fail and the workflow continues, if the configuration file doesn't exist | |
| # for a given dataset | |
| echo "is_dataset_exists=true" >> $GITHUB_OUTPUT | |
| echo "is_config_exists=true" >> $GITHUB_OUTPUT | |
| echo "is_external=${IS_EXTERNAL}" >> $GITHUB_OUTPUT | |
| # Warn about job if dataset is Hermit and config is BERT + DIET(seq) + ResponseSelector(t2t) or Sparse + BERT + DIET(seq) + ResponseSelector(t2t) | |
| if [[ "${{ matrix.dataset }}" == "Hermit" && "${{ matrix.config }}" =~ "BERT + DIET(seq) + ResponseSelector(t2t)" ]]; then | |
| echo "::warning::This ${{ matrix.dataset }} dataset / ${{ matrix.config }} config is currently being skipped on scheduled tests due to OOM associated with the upgrade to TF 2.6. You may see OOM here." | |
| fi | |
| if [[ "${IS_EXTERNAL}" == "true" ]]; then | |
| echo "DATASET_DIR=dataset_external" >> $GITHUB_ENV | |
| else | |
| echo "DATASET_DIR=dataset" >> $GITHUB_ENV | |
| test -d dataset/$DATASET || (echo "::warning::The ${{ matrix.dataset }} dataset doesn't exist. Skipping the job." \ | |
| && echo "is_config_exists=false" >> $GITHUB_OUTPUT && exit 0) | |
| fi | |
| # Skip job if a given type is not available for a given dataset | |
| if [[ -z "${DOMAIN_FILE}" && "${{ matrix.type }}" == "core" ]]; then | |
| echo "::warning::The ${{ matrix.dataset }} dataset doesn't include core type. Skipping the job." \ | |
| && echo "is_config_exists=false" >> $GITHUB_OUTPUT && exit 0 | |
| fi | |
| test -f dataset/configs/$CONFIG || (echo "::warning::The ${{ matrix.config }} configuration file doesn't exist. Skipping the job." \ | |
| && echo "is_dataset_exists=false" >> $GITHUB_OUTPUT && exit 0) | |
| echo "DATASET=${DATASET}" >> $GITHUB_ENV | |
| echo "CONFIG=${CONFIG}" >> $GITHUB_ENV | |
| echo "DOMAIN_FILE=${DOMAIN_FILE}" >> $GITHUB_ENV | |
| echo "EXTERNAL_DATASET_REPOSITORY_BRANCH=${EXTERNAL_DATASET_REPOSITORY_BRANCH}" >> $GITHUB_ENV | |
| echo "IS_EXTERNAL=${IS_EXTERNAL}" >> $GITHUB_ENV | |
| if [[ -z "${TRAIN_DIR}" ]]; then | |
| echo "TRAIN_DIR=train" >> $GITHUB_ENV | |
| else | |
| echo "TRAIN_DIR=${TRAIN_DIR}" >> $GITHUB_ENV | |
| fi | |
| if [[ -z "${TEST_DIR}" ]]; then | |
| echo "TEST_DIR=test" >> $GITHUB_ENV | |
| else | |
| echo "TEST_DIR=${TEST_DIR}" >> $GITHUB_ENV | |
| fi | |
| HOST_NAME=`hostname` | |
| echo "HOST_NAME=${HOST_NAME}" >> $GITHUB_ENV | |
| - name: Checkout dataset - external | |
| uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c | |
| if: steps.set_dataset_config_vars.outputs.is_external == 'true' | |
| with: | |
| repository: ${{ env.DATASET }} | |
| token: ${{ secrets.ML_TEST_SA_PAT }} | |
| path: "dataset_external" | |
| ref: ${{ env.EXTERNAL_DATASET_REPOSITORY_BRANCH }} | |
| - name: Set dataset commit | |
| id: set-dataset-commit | |
| working-directory: ${{ env.DATASET_DIR }} | |
| run: | | |
| DATASET_COMMIT=$(git rev-parse HEAD) | |
| echo $DATASET_COMMIT | |
| echo "dataset_commit=$DATASET_COMMIT" >> $GITHUB_OUTPUT | |
| - name: Start Datadog Agent | |
| if: steps.set_dataset_config_vars.outputs.is_dataset_exists == 'true' && steps.set_dataset_config_vars.outputs.is_config_exists == 'true' | |
| env: | |
| DATASET_NAME: "${{ matrix.dataset }}" | |
| CONFIG: "${{ matrix.config }}" | |
| DATASET_COMMIT: "${{ steps.set-dataset-commit.outputs.dataset_commit }}" | |
| BRANCH: ${{ github.head_ref }} | |
| GITHUB_SHA: "${{ github.sha }}" | |
| PR_ID: "${{ github.event.number }}" | |
| TYPE: "${{ matrix.type }}" | |
| DATASET_REPOSITORY_BRANCH: ${{ needs.read_test_configuration.outputs.dataset_branch }} | |
| INDEX_REPETITION: "${{ matrix.index_repetition }}" | |
| run: | | |
| export PR_URL="https://github.com/${GITHUB_REPOSITORY}/pull/${{ github.event.number }}" | |
| .github/scripts/start_dd_agent.sh "${{ secrets.DD_API_KEY }}" "${{ env.ACCELERATOR_TYPE }}" ${{ env.NVML_INTERVAL_IN_SEC }} | |
| - name: Set up Python 3.10 🐍 | |
| uses: actions/setup-python@b55428b1882923874294fa556849718a1d7f2ca5 | |
| if: steps.set_dataset_config_vars.outputs.is_dataset_exists == 'true' && steps.set_dataset_config_vars.outputs.is_config_exists == 'true' | |
| with: | |
| python-version: '3.10' | |
| - name: Read Poetry Version 🔢 | |
| if: steps.set_dataset_config_vars.outputs.is_dataset_exists == 'true' && steps.set_dataset_config_vars.outputs.is_config_exists == 'true' | |
| run: | | |
| echo "POETRY_VERSION=$(scripts/poetry-version.sh)" >> $GITHUB_ENV | |
| shell: bash | |
| - name: Install poetry 🦄 | |
| uses: Gr1N/setup-poetry@15821dc8a61bc630db542ae4baf6a7c19a994844 # v8 | |
| if: steps.set_dataset_config_vars.outputs.is_dataset_exists == 'true' && steps.set_dataset_config_vars.outputs.is_config_exists == 'true' | |
| with: | |
| poetry-version: ${{ env.POETRY_VERSION }} | |
| - name: Load Poetry Cached Libraries ⬇ | |
| uses: actions/cache@6998d139ddd3e68c71e9e398d8e40b71a2f39812 | |
| if: steps.set_dataset_config_vars.outputs.is_dataset_exists == 'true' && steps.set_dataset_config_vars.outputs.is_config_exists == 'true' | |
| with: | |
| path: ~/.cache/pypoetry/virtualenvs | |
| key: ${{ runner.os }}-poetry-${{ env.POETRY_VERSION }}-3.9-${{ hashFiles('**/poetry.lock') }}-${{ secrets.POETRY_CACHE_VERSION }} | |
| - name: Install Dependencies 📦 | |
| if: steps.set_dataset_config_vars.outputs.is_dataset_exists == 'true' && steps.set_dataset_config_vars.outputs.is_config_exists == 'true' | |
| run: | | |
| make install-full | |
| poetry run python -m spacy download de_core_news_md | |
| - name: Install datadog dependencies | |
| if: steps.set_dataset_config_vars.outputs.is_dataset_exists == 'true' && steps.set_dataset_config_vars.outputs.is_config_exists == 'true' | |
| run: poetry run pip install -U datadog-api-client ddtrace | |
| - name: Validate that GPUs are working | |
| if: steps.set_dataset_config_vars.outputs.is_dataset_exists == 'true' && steps.set_dataset_config_vars.outputs.is_config_exists == 'true' | |
| run: |- | |
| poetry run python .github/scripts/validate_gpus.py | |
| - name: Download pretrained models 💪 | |
| if: steps.set_dataset_config_vars.outputs.is_dataset_exists == 'true' && steps.set_dataset_config_vars.outputs.is_config_exists == 'true' | |
| run: |- | |
| poetry run python .github/scripts/download_pretrained.py --config dataset/configs/${CONFIG} | |
| - name: Run test | |
| id: run_test | |
| if: steps.set_dataset_config_vars.outputs.is_dataset_exists == 'true' && steps.set_dataset_config_vars.outputs.is_config_exists == 'true' | |
| env: | |
| TFHUB_CACHE_DIR: ~/.tfhub_cache/ | |
| OMP_NUM_THREADS: 1 | |
| run: |- | |
| poetry run rasa --version | |
| export NOW_TRAIN=$(gomplate -i '{{ (time.Now).Format time.RFC3339}}'); | |
| cd ${{ github.workspace }} | |
| if [[ "${{ steps.set_dataset_config_vars.outputs.is_external }}" == "true" ]]; then | |
| export DATASET=. | |
| fi | |
| if [[ "${{ matrix.type }}" == "nlu" ]]; then | |
| poetry run ddtrace-run rasa train nlu --quiet -u ${DATASET_DIR}/${DATASET}/${TRAIN_DIR} -c dataset/configs/${CONFIG} --out ${DATASET_DIR}/models/${DATASET}/${CONFIG} | |
| echo "train_run_time=$(gomplate -i '{{ $t := time.Parse time.RFC3339 (getenv "NOW_TRAIN") }}{{ (time.Since $t).Round (time.Second 1) }}')" >> $GITHUB_OUTPUT | |
| export NOW_TEST=$(gomplate -i '{{ (time.Now).Format time.RFC3339}}'); | |
| poetry run ddtrace-run rasa test nlu --quiet -u ${DATASET_DIR}/$DATASET/${TEST_DIR} -m ${DATASET_DIR}/models/$DATASET/$CONFIG --out ${{ github.workspace }}/results/$DATASET/$CONFIG | |
| echo "test_run_time=$(gomplate -i '{{ $t := time.Parse time.RFC3339 (getenv "NOW_TEST") }}{{ (time.Since $t).Round (time.Second 1) }}')" >> $GITHUB_OUTPUT | |
| echo "total_run_time=$(gomplate -i '{{ $t := time.Parse time.RFC3339 (getenv "NOW_TRAIN") }}{{ (time.Since $t).Round (time.Second 1) }}')" >> $GITHUB_OUTPUT | |
| elif [[ "${{ matrix.type }}" == "core" ]]; then | |
| poetry run ddtrace-run rasa train core --quiet -s ${DATASET_DIR}/$DATASET/$TRAIN_DIR -c dataset/configs/$CONFIG -d ${DATASET_DIR}/${DATASET}/${DOMAIN_FILE} | |
| echo "train_run_time=$(gomplate -i '{{ $t := time.Parse time.RFC3339 (getenv "NOW_TRAIN") }}{{ (time.Since $t).Round (time.Second 1) }}')" >> $GITHUB_OUTPUT | |
| export NOW_TEST=$(gomplate -i '{{ (time.Now).Format time.RFC3339}}'); | |
| poetry run ddtrace-run rasa test core -s ${DATASET_DIR}/${DATASET}/${TEST_DIR} --out ${{ github.workspace }}/results/${{ matrix.dataset }}/${CONFIG} | |
| echo "test_run_time=$(gomplate -i '{{ $t := time.Parse time.RFC3339 (getenv "NOW_TEST") }}{{ (time.Since $t).Round (time.Second 1) }}')" >> $GITHUB_OUTPUT | |
| echo "total_run_time=$(gomplate -i '{{ $t := time.Parse time.RFC3339 (getenv "NOW_TRAIN") }}{{ (time.Since $t).Round (time.Second 1) }}')" >> $GITHUB_OUTPUT | |
| fi | |
| - name: Generate a JSON file with a report / Publish results to Datadog | |
| if: steps.set_dataset_config_vars.outputs.is_dataset_exists == 'true' && steps.set_dataset_config_vars.outputs.is_config_exists == 'true' | |
| env: | |
| SUMMARY_FILE: "./report.json" | |
| DATASET_NAME: ${{ matrix.dataset }} | |
| RESULT_DIR: "${{ github.workspace }}/results" | |
| CONFIG: ${{ matrix.config }} | |
| TEST_RUN_TIME: ${{ steps.run_test.outputs.test_run_time }} | |
| TRAIN_RUN_TIME: ${{ steps.run_test.outputs.train_run_time }} | |
| TOTAL_RUN_TIME: ${{ steps.run_test.outputs.total_run_time }} | |
| DATASET_REPOSITORY_BRANCH: ${{ needs.read_test_configuration.outputs.dataset_branch }} | |
| TYPE: ${{ matrix.type }} | |
| INDEX_REPETITION: ${{ matrix.index_repetition }} | |
| DATASET_COMMIT: ${{ steps.set-dataset-commit.outputs.dataset_commit }} | |
| BRANCH: ${{ github.head_ref }} | |
| GITHUB_SHA: "${{ github.sha }}" | |
| PR_ID: "${{ github.event.number }}" | |
| DD_APP_KEY: ${{ secrets.DD_APP_KEY_PERF_TEST }} | |
| DD_API_KEY: ${{ secrets.DD_API_KEY }} | |
| DD_SITE: datadoghq.eu | |
| run: |- | |
| export PR_URL="https://github.com/${GITHUB_REPOSITORY}/pull/${{ github.event.number }}" | |
| poetry run pip install analytics-python | |
| poetry run python .github/scripts/mr_publish_results.py | |
| cat $SUMMARY_FILE | |
| - name: Upload an artifact with the report | |
| uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce | |
| if: steps.set_dataset_config_vars.outputs.is_dataset_exists == 'true' && steps.set_dataset_config_vars.outputs.is_config_exists == 'true' | |
| with: | |
| name: report-${{ matrix.dataset }}-${{ matrix.config }}-${{ matrix.index_repetition }} | |
| path: report.json | |
| - name: Stop Datadog Agent | |
| if: ${{ always() && steps.set_dataset_config_vars.outputs.is_dataset_exists == 'true' && steps.set_dataset_config_vars.outputs.is_config_exists == 'true' }} | |
| run: | | |
| sudo service datadog-agent stop | |
| model_regression_test_cpu: | |
| name: Model Regression Tests - CPU | |
| needs: | |
| - read_test_configuration | |
| env: | |
| ACCELERATOR_TYPE: "CPU" | |
| runs-on: ubuntu-22.04 | |
| strategy: | |
| max-parallel: 3 | |
| matrix: ${{fromJson(needs.read_test_configuration.outputs.matrix)}} | |
| fail-fast: false | |
| if: ${{ !contains(github.event.pull_request.labels.*.name, 'runner:gpu') && github.repository == 'RasaHQ/rasa' && contains(github.event.pull_request.labels.*.name, 'status:model-regression-tests') && needs.read_test_configuration.outputs.configuration_id != '' }} | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c | |
| - name: Checkout dataset | |
| uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c | |
| with: | |
| repository: ${{ secrets.DATASET_REPOSITORY }} | |
| token: ${{ secrets.ML_TEST_SA_PAT }} | |
| path: "dataset" | |
| ref: ${{ needs.read_test_configuration.outputs.dataset_branch }} | |
| - name: Download gomplate | |
| run: |- | |
| sudo curl -o /usr/local/bin/gomplate -sSL https://github.com/hairyhenderson/gomplate/releases/download/v3.9.0/gomplate_linux-amd64 | |
| sudo chmod +x /usr/local/bin/gomplate | |
| - name: Set env variables | |
| id: set_dataset_config_vars | |
| env: | |
| DATASET_NAME: "${{ matrix.dataset }}" | |
| CONFIG_NAME: "${{ matrix.config }}" | |
| run: |- | |
| # determine extra environment variables | |
| # - CONFIG | |
| # - DATASET | |
| # - IS_EXTERNAL | |
| # - EXTERNAL_DATASET_REPOSITORY_BRANCH | |
| # - TRAIN_DIR | |
| # - TEST_DIR | |
| # - DOMAIN_FILE | |
| source <(gomplate -d mapping=./dataset/dataset_config_mapping.json -f .github/templates/configuration_variables.tmpl) | |
| # Not all configurations are available for all datasets. | |
| # The job will fail and the workflow continues, if the configuration file doesn't exist | |
| # for a given dataset | |
| echo "is_dataset_exists=true" >> $GITHUB_OUTPUT | |
| echo "is_config_exists=true" >> $GITHUB_OUTPUT | |
| echo "is_external=${IS_EXTERNAL}" >> $GITHUB_OUTPUT | |
| if [[ "${IS_EXTERNAL}" == "true" ]]; then | |
| echo "DATASET_DIR=dataset_external" >> $GITHUB_ENV | |
| else | |
| echo "DATASET_DIR=dataset" >> $GITHUB_ENV | |
| test -d dataset/$DATASET || (echo "::warning::The ${{ matrix.dataset }} dataset doesn't exist. Skipping the job." \ | |
| && echo "is_config_exists=false" >> $GITHUB_OUTPUT && exit 0) | |
| fi | |
| # Skip job if a given type is not available for a given dataset | |
| if [[ -z "${DOMAIN_FILE}" && "${{ matrix.type }}" == "core" ]]; then | |
| echo "::warning::The ${{ matrix.dataset }} dataset doesn't include core type. Skipping the job." \ | |
| && echo "is_config_exists=false" >> $GITHUB_OUTPUT && exit 0 | |
| fi | |
| test -f dataset/configs/$CONFIG || (echo "::warning::The ${{ matrix.config }} configuration file doesn't exist. Skipping the job." \ | |
| && echo "is_dataset_exists=false" >> $GITHUB_OUTPUT && exit 0) | |
| echo "DATASET=${DATASET}" >> $GITHUB_ENV | |
| echo "CONFIG=${CONFIG}" >> $GITHUB_ENV | |
| echo "DOMAIN_FILE=${DOMAIN_FILE}" >> $GITHUB_ENV | |
| echo "EXTERNAL_DATASET_REPOSITORY_BRANCH=${EXTERNAL_DATASET_REPOSITORY_BRANCH}" >> $GITHUB_ENV | |
| echo "IS_EXTERNAL=${IS_EXTERNAL}" >> $GITHUB_ENV | |
| if [[ -z "${TRAIN_DIR}" ]]; then | |
| echo "TRAIN_DIR=train" >> $GITHUB_ENV | |
| else | |
| echo "TRAIN_DIR=${TRAIN_DIR}" >> $GITHUB_ENV | |
| fi | |
| if [[ -z "${TEST_DIR}" ]]; then | |
| echo "TEST_DIR=test" >> $GITHUB_ENV | |
| else | |
| echo "TEST_DIR=${TEST_DIR}" >> $GITHUB_ENV | |
| fi | |
| HOST_NAME=`hostname` | |
| echo "HOST_NAME=${HOST_NAME}" >> $GITHUB_ENV | |
| - name: Checkout dataset - external | |
| uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c | |
| if: steps.set_dataset_config_vars.outputs.is_external == 'true' | |
| with: | |
| repository: ${{ env.DATASET }} | |
| token: ${{ secrets.ML_TEST_SA_PAT }} | |
| path: "dataset_external" | |
| ref: ${{ env.EXTERNAL_DATASET_REPOSITORY_BRANCH }} | |
| - name: Set dataset commit | |
| id: set-dataset-commit | |
| working-directory: ${{ env.DATASET_DIR }} | |
| run: | | |
| DATASET_COMMIT=$(git rev-parse HEAD) | |
| echo $DATASET_COMMIT | |
| echo "dataset_commit=$DATASET_COMMIT" >> $GITHUB_OUTPUT | |
| - name: Start Datadog Agent | |
| if: steps.set_dataset_config_vars.outputs.is_dataset_exists == 'true' && steps.set_dataset_config_vars.outputs.is_config_exists == 'true' | |
| env: | |
| DATASET_NAME: "${{ matrix.dataset }}" | |
| CONFIG: "${{ matrix.config }}" | |
| DATASET_COMMIT: "${{ steps.set-dataset-commit.outputs.dataset_commit }}" | |
| BRANCH: ${{ github.head_ref }} | |
| GITHUB_SHA: "${{ github.sha }}" | |
| PR_ID: "${{ github.event.number }}" | |
| TYPE: "${{ matrix.type }}" | |
| DATASET_REPOSITORY_BRANCH: ${{ matrix.dataset_branch }} | |
| INDEX_REPETITION: "${{ matrix.index_repetition }}" | |
| run: | | |
| export PR_URL="https://github.com/${GITHUB_REPOSITORY}/pull/${{ github.event.number }}" | |
| .github/scripts/start_dd_agent.sh "${{ secrets.DD_API_KEY }}" "${{ env.ACCELERATOR_TYPE }}" ${{ env.NVML_INTERVAL_IN_SEC }} | |
| - name: Set up Python 3.10 🐍 | |
| uses: actions/setup-python@b55428b1882923874294fa556849718a1d7f2ca5 | |
| if: steps.set_dataset_config_vars.outputs.is_dataset_exists == 'true' && steps.set_dataset_config_vars.outputs.is_config_exists == 'true' | |
| with: | |
| python-version: '3.10' | |
| - name: Read Poetry Version 🔢 | |
| if: steps.set_dataset_config_vars.outputs.is_dataset_exists == 'true' && steps.set_dataset_config_vars.outputs.is_config_exists == 'true' | |
| run: | | |
| echo "POETRY_VERSION=$(scripts/poetry-version.sh)" >> $GITHUB_ENV | |
| shell: bash | |
| - name: Install poetry 🦄 | |
| uses: Gr1N/setup-poetry@15821dc8a61bc630db542ae4baf6a7c19a994844 # v8 | |
| if: steps.set_dataset_config_vars.outputs.is_dataset_exists == 'true' && steps.set_dataset_config_vars.outputs.is_config_exists == 'true' | |
| with: | |
| poetry-version: ${{ env.POETRY_VERSION }} | |
| - name: Load Poetry Cached Libraries ⬇ | |
| uses: actions/cache@6998d139ddd3e68c71e9e398d8e40b71a2f39812 | |
| if: steps.set_dataset_config_vars.outputs.is_dataset_exists == 'true' && steps.set_dataset_config_vars.outputs.is_config_exists == 'true' | |
| with: | |
| path: ~/.cache/pypoetry/virtualenvs | |
| key: ${{ runner.os }}-poetry-${{ env.POETRY_VERSION }}-3.9-${{ hashFiles('**/poetry.lock') }}-${{ secrets.POETRY_CACHE_VERSION }} | |
| - name: Install Dependencies 📦 | |
| if: steps.set_dataset_config_vars.outputs.is_dataset_exists == 'true' && steps.set_dataset_config_vars.outputs.is_config_exists == 'true' | |
| run: | | |
| make install-full | |
| poetry run python -m spacy download de_core_news_md | |
| - name: Install datadog dependencies | |
| if: steps.set_dataset_config_vars.outputs.is_dataset_exists == 'true' && steps.set_dataset_config_vars.outputs.is_config_exists == 'true' | |
| run: poetry run pip install -U datadog-api-client ddtrace | |
| - name: CPU run - Validate that no GPUs are available | |
| if: steps.set_dataset_config_vars.outputs.is_dataset_exists == 'true' && steps.set_dataset_config_vars.outputs.is_config_exists == 'true' | |
| run: |- | |
| poetry run python .github/scripts/validate_cpu.py | |
| - name: Download pretrained models 💪 | |
| if: steps.set_dataset_config_vars.outputs.is_dataset_exists == 'true' && steps.set_dataset_config_vars.outputs.is_config_exists == 'true' | |
| run: |- | |
| poetry run python .github/scripts/download_pretrained.py --config dataset/configs/${CONFIG} | |
| - name: Run test | |
| id: run_test | |
| if: steps.set_dataset_config_vars.outputs.is_dataset_exists == 'true' && steps.set_dataset_config_vars.outputs.is_config_exists == 'true' | |
| env: | |
| TFHUB_CACHE_DIR: ~/.tfhub_cache/ | |
| OMP_NUM_THREADS: 1 | |
| run: |- | |
| poetry run rasa --version | |
| export NOW_TRAIN=$(gomplate -i '{{ (time.Now).Format time.RFC3339}}'); | |
| cd ${{ github.workspace }} | |
| if [[ "${{ steps.set_dataset_config_vars.outputs.is_external }}" == "true" ]]; then | |
| export DATASET=. | |
| fi | |
| if [[ "${{ matrix.type }}" == "nlu" ]]; then | |
| poetry run ddtrace-run rasa train nlu --quiet -u ${DATASET_DIR}/${DATASET}/${TRAIN_DIR} -c dataset/configs/${CONFIG} --out ${DATASET_DIR}/models/${DATASET}/${CONFIG} | |
| echo "train_run_time=$(gomplate -i '{{ $t := time.Parse time.RFC3339 (getenv "NOW_TRAIN") }}{{ (time.Since $t).Round (time.Second 1) }}')" >> $GITHUB_OUTPUT | |
| export NOW_TEST=$(gomplate -i '{{ (time.Now).Format time.RFC3339}}'); | |
| poetry run ddtrace-run rasa test nlu --quiet -u ${DATASET_DIR}/$DATASET/${TEST_DIR} -m ${DATASET_DIR}/models/$DATASET/$CONFIG --out ${{ github.workspace }}/results/$DATASET/$CONFIG | |
| echo "test_run_time=$(gomplate -i '{{ $t := time.Parse time.RFC3339 (getenv "NOW_TEST") }}{{ (time.Since $t).Round (time.Second 1) }}')" >> $GITHUB_OUTPUT | |
| echo "total_run_time=$(gomplate -i '{{ $t := time.Parse time.RFC3339 (getenv "NOW_TRAIN") }}{{ (time.Since $t).Round (time.Second 1) }}')" >> $GITHUB_OUTPUT | |
| elif [[ "${{ matrix.type }}" == "core" ]]; then | |
| poetry run ddtrace-run rasa train core --quiet -s ${DATASET_DIR}/$DATASET/$TRAIN_DIR -c dataset/configs/$CONFIG -d ${DATASET_DIR}/${DATASET}/${DOMAIN_FILE} | |
| echo "train_run_time=$(gomplate -i '{{ $t := time.Parse time.RFC3339 (getenv "NOW_TRAIN") }}{{ (time.Since $t).Round (time.Second 1) }}')" >> $GITHUB_OUTPUT | |
| export NOW_TEST=$(gomplate -i '{{ (time.Now).Format time.RFC3339}}'); | |
| poetry run ddtrace-run rasa test core -s ${DATASET_DIR}/${DATASET}/${TEST_DIR} --out ${{ github.workspace }}/results/${{ matrix.dataset }}/${CONFIG} | |
| echo "test_run_time=$(gomplate -i '{{ $t := time.Parse time.RFC3339 (getenv "NOW_TEST") }}{{ (time.Since $t).Round (time.Second 1) }}')" >> $GITHUB_OUTPUT | |
| echo "total_run_time=$(gomplate -i '{{ $t := time.Parse time.RFC3339 (getenv "NOW_TRAIN") }}{{ (time.Since $t).Round (time.Second 1) }}')" >> $GITHUB_OUTPUT | |
| fi | |
| - name: Generate a JSON file with a report / Publish results to Datadog | |
| if: steps.set_dataset_config_vars.outputs.is_dataset_exists == 'true' && steps.set_dataset_config_vars.outputs.is_config_exists == 'true' | |
| env: | |
| SUMMARY_FILE: "./report.json" | |
| DATASET_NAME: ${{ matrix.dataset }} | |
| RESULT_DIR: "${{ github.workspace }}/results" | |
| CONFIG: ${{ matrix.config }} | |
| TEST_RUN_TIME: ${{ steps.run_test.outputs.test_run_time }} | |
| TRAIN_RUN_TIME: ${{ steps.run_test.outputs.train_run_time }} | |
| TOTAL_RUN_TIME: ${{ steps.run_test.outputs.total_run_time }} | |
| DATASET_REPOSITORY_BRANCH: ${{ needs.read_test_configuration.outputs.dataset_branch }} | |
| TYPE: ${{ matrix.type }} | |
| INDEX_REPETITION: ${{ matrix.index_repetition }} | |
| DATASET_COMMIT: ${{ steps.set-dataset-commit.outputs.dataset_commit }} | |
| BRANCH: ${{ github.head_ref }} | |
| GITHUB_SHA: "${{ github.sha }}" | |
| PR_ID: "${{ github.event.number }}" | |
| DD_APP_KEY: ${{ secrets.DD_APP_KEY_PERF_TEST }} | |
| DD_API_KEY: ${{ secrets.DD_API_KEY }} | |
| DD_SITE: datadoghq.eu | |
| run: |- | |
| export PR_URL="https://github.com/${GITHUB_REPOSITORY}/pull/${{ github.event.number }}" | |
| poetry run pip install analytics-python | |
| poetry run python .github/scripts/mr_publish_results.py | |
| cat $SUMMARY_FILE | |
| - name: Upload an artifact with the report | |
| uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce | |
| if: steps.set_dataset_config_vars.outputs.is_dataset_exists == 'true' && steps.set_dataset_config_vars.outputs.is_config_exists == 'true' | |
| with: | |
| name: report-${{ matrix.dataset }}-${{ matrix.config }}-${{ matrix.index_repetition }} | |
| path: report.json | |
| - name: Stop Datadog Agent | |
| if: ${{ always() && steps.set_dataset_config_vars.outputs.is_dataset_exists == 'true' && steps.set_dataset_config_vars.outputs.is_config_exists == 'true' }} | |
| run: | | |
| sudo service datadog-agent stop | |
| combine_reports: | |
| name: Combine reports | |
| runs-on: ubuntu-22.04 | |
| needs: | |
| - model_regression_test_cpu | |
| - model_regression_test_gpu | |
| if: ${{ always() && ((needs.model_regression_test_cpu.result != 'skipped') != (needs.model_regression_test_gpu.result != 'skipped')) }} | |
| outputs: | |
| success_status: ${{ steps.set-success-status.outputs.success_status }} | |
| steps: | |
| - name: Set success status | |
| id: set-success-status | |
| run: |- | |
| succeeded=${{ needs.model_regression_test_cpu.result == 'success' || needs.model_regression_test_gpu.result == 'success' }} | |
| if [[ $succeeded == "false" ]]; then | |
| success_status="Failed" | |
| elif [[ $succeeded == "true" ]]; then | |
| success_status="Succeeded" | |
| else | |
| success_status="Unknown" | |
| fi | |
| echo $success_status | |
| echo "success_status=$success_status" >> $GITHUB_OUTPUT | |
| - name: Checkout git repository 🕝 | |
| uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c | |
| - name: Set up Python 3.10 🐍 | |
| uses: actions/setup-python@b55428b1882923874294fa556849718a1d7f2ca5 | |
| with: | |
| python-version: '3.10' | |
| - name: Get reports | |
| uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a | |
| with: | |
| path: reports/ | |
| - name: Display structure of downloaded files | |
| continue-on-error: true | |
| run: ls -R | |
| working-directory: reports/ | |
| - name: Merge all reports | |
| env: | |
| SUMMARY_FILE: "./report.json" | |
| REPORTS_DIR: "reports/" | |
| run: | | |
| python .github/scripts/mr_generate_summary.py | |
| cat $SUMMARY_FILE | |
| - name: Upload an artifact with the overall report | |
| uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce | |
| with: | |
| name: report.json | |
| path: ./report.json | |
| set_job_success_status: | |
| name: Set job success status | |
| runs-on: ubuntu-22.04 | |
| needs: | |
| - combine_reports | |
| if: ${{ always() && needs.combine_reports.result == 'success' }} | |
| steps: | |
| - name: Set return code | |
| run: | | |
| success_status=${{ needs.combine_reports.outputs.success_status }} | |
| echo "Status: $success_status" | |
| if [[ $success_status == "Succeeded" ]]; then | |
| exit 0 | |
| else | |
| exit 1 | |
| fi | |
| add_comment_results: | |
| name: Add a comment with the results | |
| runs-on: ubuntu-22.04 | |
| needs: | |
| - combine_reports | |
| if: ${{ always() && needs.combine_reports.result == 'success' }} | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c | |
| - name: Download report from last on-schedule regression test | |
| run: | | |
| # Get ID of last on-schedule workflow | |
| SCHEDULE_ID=$(curl -X GET -s -H 'Authorization: token ${{ secrets.GITHUB_TOKEN }}' -H "Accept: application/vnd.github.v3+json" \ | |
| "https://api.github.com/repos/${{ github.repository }}/actions/workflows" \ | |
| | jq -r '.workflows[] | select(.name == "CI - Model Regression on schedule") | select(.path | test("schedule")) | .id') | |
| ARTIFACT_URL=$(curl -s -H 'Authorization: token ${{ secrets.GITHUB_TOKEN }}' -H "Accept: application/vnd.github.v3+json" \ | |
| "https://api.github.com/repos/${{ github.repository }}/actions/workflows/${SCHEDULE_ID}/runs?event=schedule&status=completed&branch=main&per_page=1" | jq -r .workflow_runs[0].artifacts_url) | |
| DOWNLOAD_URL=$(curl -s -H 'Authorization: token ${{ secrets.GITHUB_TOKEN }}' -H "Accept: application/vnd.github.v3+json" "${ARTIFACT_URL}" \ | |
| | jq -r '.artifacts[] | select(.name == "report.json") | .archive_download_url') | |
| # Download the artifact | |
| curl -H 'Authorization: token ${{ secrets.GITHUB_TOKEN }}' -LJO -H "Accept: application/vnd.github.v3+json" $DOWNLOAD_URL | |
| # Unzip and change name | |
| unzip report.json.zip && mv report.json report_main.json | |
| - name: Download the report | |
| uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a | |
| with: | |
| name: report.json | |
| - name: Download gomplate | |
| run: |- | |
| sudo curl -o /usr/local/bin/gomplate -sSL https://github.com/hairyhenderson/gomplate/releases/download/v3.9.0/gomplate_linux-amd64 | |
| sudo chmod +x /usr/local/bin/gomplate | |
| - name: Render a comment to add | |
| id: get_results | |
| run: | | |
| OUTPUT="$(gomplate -d data=report.json -d results_main=report_main.json -f .github/templates/model_regression_test_results.tmpl)" | |
| OUTPUT="${OUTPUT//$'\n'/'%0A'}" | |
| OUTPUT="${OUTPUT//$'\r'/'%0D'}" | |
| echo "result=$OUTPUT" >> $GITHUB_OUTPUT | |
| # Get time of current commit as start time | |
| TIME_ISO_COMMIT=$(gomplate -d github=https://api.github.com/repos/rasaHQ/rasa/commits/${{ github.sha }} -H 'github=Authorization:token ${{ secrets.GITHUB_TOKEN }}' -i '{{ (ds "github").commit.author.date }}') # Example "2022-02-17T14:06:38Z" | |
| TIME_UNIX_COMMIT=$(date -d "${TIME_ISO_COMMIT}" +%s%3N) # Example: "1645106798" | |
| # Get current time | |
| TIME_ISO_NOW=$(gomplate -i '{{ (time.Now).UTC.Format time.RFC3339}}') # Example: "2022-02-17T14:50:54Z%" | |
| TIME_UNIX_NOW=$(date -d "${TIME_ISO_NOW}" +%s%3N) # Example: "1645118083" | |
| echo "from_ts=$TIME_UNIX_COMMIT" >> $GITHUB_OUTPUT | |
| echo "to_ts=$TIME_UNIX_NOW" >> $GITHUB_OUTPUT | |
| - name: Publish results as a PR comment | |
| uses: marocchino/sticky-pull-request-comment@fcf6fe9e4a0409cd9316a5011435be0f3327f1e1 # v2.3.1 | |
| if: ${{ always() }} | |
| with: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| header: ${{ github.run_id }} | |
| append: true | |
| message: |- | |
| Status of the run: ${{ needs.combine_reports.outputs.success_status }} | |
| Commit: ${{ github.sha }}, [The full report is available as an artifact.](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}) | |
| [Datadog dashboard link](https://app.datadoghq.eu/dashboard/mf4-2hu-x84?tpl_var_branch_baseline=${{ github.head_ref }}&from_ts=${{ steps.get_results.outputs.from_ts }}&to_ts=${{ steps.get_results.outputs.to_ts }}&live=false) | |
| ${{ steps.get_results.outputs.result }} | |
| - name: Remove 'status:model-regression-tests' label | |
| continue-on-error: true | |
| uses: buildsville/add-remove-label@6008d7bd99d3baeb7c04033584e68f8ec80b198b # v1.0 | |
| with: | |
| token: ${{secrets.GITHUB_TOKEN}} | |
| label: "status:model-regression-tests" | |
| type: remove | |
| - name: Remove 'runner:gpu' label | |
| continue-on-error: true | |
| uses: buildsville/add-remove-label@6008d7bd99d3baeb7c04033584e68f8ec80b198b # v1.0 | |
| with: | |
| token: ${{secrets.GITHUB_TOKEN}} | |
| label: "runner:gpu" | |
| type: remove | |
| remove_runner_gpu: | |
| name: Delete Github Runner - GPU | |
| needs: | |
| - deploy_runner_gpu | |
| - model_regression_test_gpu | |
| runs-on: ubuntu-22.04 | |
| if: ${{ always() && needs.deploy_runner_gpu.result != 'skipped' && contains(github.event.pull_request.labels.*.name, 'runner:gpu') && contains(github.event.pull_request.labels.*.name, 'status:model-regression-tests') }} | |
| steps: | |
| # Setup gcloud CLI | |
| - uses: google-github-actions/setup-gcloud@877d4953d2c70a0ba7ef3290ae968eb24af233bb | |
| with: | |
| version: "${{ env.GCLOUD_VERSION }}" | |
| service_account_key: ${{ secrets.GKE_SA_RASA_CI_CD_GPU_RASA_CI_CD }} | |
| service_account_email: ${{ secrets.GKE_RASA_CI_GPU_SA_NAME_RASA_CI_CD }} | |
| # Get the GKE credentials so we can deploy to the cluster | |
| - run: |- | |
| gcloud container clusters get-credentials "${{ secrets.GKE_GPU_CLUSTER_RASA_CI_CD }}" --zone "$GKE_ZONE" --project "${{ secrets.GKE_SA_RASA_CI_GPU_PROJECT_RASA_CI_CD }}" | |
| - name: Remove Github Runner | |
| run: kubectl -n github-runner delete deployments github-runner-${GITHUB_RUN_ID} --grace-period=30 |