Skip to content

Commit

Permalink
Circleci pipeline change (#66)
Browse files Browse the repository at this point in the history 
Moved tests running to the vizro-qa pipeline
  • Loading branch information
@l0uden
l0uden authored Sep 26, 2023
1 parent 583e951 commit 4303ae0
Show file tree
Hide file tree
Showing 2 changed files with 104 additions and 212 deletions.
274 changes: 62 additions & 212 deletions .circleci/config.yml
View file
Original file line number Diff line number Diff line change
@@ -1,232 +1,82 @@
version: 2.1

orbs:
browser-tools: circleci/[email protected]

parameters:
triggering-pipeline-id:
type: string
default: ""
branch:
type: string
default: ""
qa_branch:
type: string
default: ""
app_repo_name:
type: string
default: "vizro"
app_core_name:
type: string
default: "vizro-core"
qa_repo_name:
type: string
default: "vizro-qa"

commands:
approve-job-in-triggering-pipeline:
jobs:
trigger-new-pipeline:
docker:
- image: cimg/base:2021.11
resource_class: small
steps:
- run:
name: Ping CircleCI API to get workflow
name: Ping another pipeline
command: |
echo << pipeline.parameters.triggering-pipeline-id >>
if ! [ -z "<< pipeline.parameters.triggering-pipeline-id >>" ]
then
workflow_id=$(curl --request GET \
--url https://circleci.com/api/v2/pipeline/<< pipeline.parameters.triggering-pipeline-id >>/workflow \
--header "Circle-Token: $CIRCLECI_API_KEY" \
--header "content-type: application/json" \
| jq -r '.items[0].id')
echo $workflow_id
waiting_job_id=$(curl --request GET \
--url https://circleci.com/api/v2/workflow/$workflow_id/job \
--header "Circle-Token: $CIRCLECI_API_KEY" \
--header "content-type: application/json" \
| jq -r '.items[] | select(.name == "wait-for-triggered-pipeline").id')
echo $waiting_job_id
curl --request POST \
--url https://circleci.com/api/v2/workflow/$workflow_id/approve/$waiting_job_id \
--header "Circle-Token: $CIRCLECI_API_KEY" \
--header "content-type: application/json"
CREATED_PIPELINE=$(curl --request POST \
--url "$QA_PIPELINE_URL" \
--header "Circle-Token: $CIRCLECI_API_KEY" \
--header "content-type: application/json" \
--data '{"branch":"<< pipeline.git.branch >>", "parameters":{"triggering-pipeline-id":"<< pipeline.id >>", "branch":"<< pipeline.git.branch >>", "vizro_branch": "<< pipeline.git.branch >>"}}' \
| jq -r '.id'
)
echo "my created pipeline"
echo $CREATED_PIPELINE
if [[ "$CREATED_PIPELINE" == "null" ]]; then
CREATED_PIPELINE=$(curl --request POST \
--url "$QA_PIPELINE_URL" \
--header "Circle-Token: $CIRCLECI_API_KEY" \
--header "content-type: application/json" \
--data '{"branch":"main", "parameters":{"triggering-pipeline-id":"<< pipeline.id >>", "branch":"main", "vizro_branch": "<< pipeline.git.branch >>"}}' \
| jq -r '.id'
)
fi
when: always
echo "my created pipeline"
echo $CREATED_PIPELINE
mkdir ~/workspace
echo $CREATED_PIPELINE > pipeline.txt
- persist_to_workspace:
root: .
paths:
- pipeline.txt

chromedriver_install:
description: Installation of Chrome and chromedriver
parameters:
version:
default: 114.0.5735.90
type: string
check-status-of-triggered-pipeline:
docker:
- image: cimg/base:2021.11
resource_class: small
steps:
- attach_workspace:
at: workspace
- run:
name: Install Google Chrome
name: Check triggered workflow status
command: |
cd ~
sudo apt-get update
sudo apt-get install libu2f-udev -y
wget "https://dl.google.com/linux/chrome/deb/pool/main/g/google-chrome-stable/google-chrome-stable_<< parameters.version >>-1_amd64.deb"
sudo apt-get install -y --allow-downgrades ~/google-chrome-stable_<< parameters.version >>-1_amd64.deb
rm ~/google-chrome-stable_<< parameters.version >>-1_amd64.deb
- browser-tools/install-chromedriver
triggered_pipeline_id=$(cat workspace/pipeline.txt)
custom_checkout:
steps:
- run:
name: Checkout logic for qa and app repos
command: |
touch ~/.ssh/known_hosts
ssh-keyscan github.com >> ~/.ssh/known_hosts
cd /home/circleci/
git clone -v "$TESTS_REPO_URL"
if ! [ -z "<< pipeline.parameters.qa_branch >>" ]
then
echo "from pipeline"
echo "<< pipeline.parameters.qa_repo_name >> branch:"
cd /home/circleci/<< pipeline.parameters.qa_repo_name >>
git switch "<< pipeline.parameters.qa_branch >>"
git rev-parse HEAD
cd /home/circleci/project/
existed_in_remote=$(git ls-remote --heads origin "<< pipeline.parameters.qa_branch >>")
if ! [[ -z ${existed_in_remote} ]]; then
echo "<< pipeline.parameters.app_repo_name >> branch:"
git switch "<< pipeline.parameters.branch >>"
git rev-parse HEAD
fi
elif ! [ -z "$CIRCLE_BRANCH" ]
then
echo "from git"
echo "<< pipeline.parameters.app_repo_name >> branch:"
cd /home/circleci/project/
git switch "$CIRCLE_BRANCH"
git rev-parse HEAD
cd /home/circleci/<< pipeline.parameters.qa_repo_name >>
existed_in_remote=$(git ls-remote --heads origin "$CIRCLE_BRANCH")
if ! [[ -z ${existed_in_remote} ]]; then
echo "<< pipeline.parameters.qa_repo_name >> branch:"
git switch "$CIRCLE_BRANCH"
git rev-parse HEAD
fi
fi
created_workflow_status=$(curl --request GET \
--url "https://circleci.com/api/v2/pipeline/${triggered_pipeline_id}/workflow" \
--header "Circle-Token: $CIRCLECI_API_KEY" \
--header "content-type: application/json" \
| jq -r '.items[0].status'
)
create_env_and_install_reqs:
description: Create environment and install requirements
steps:
- run:
name: Create qa environment
command: |
python3 -m venv ~/.venv/<< pipeline.parameters.qa_repo_name >>
echo "source ~/.venv/<< pipeline.parameters.qa_repo_name >>/bin/activate" >> $BASH_ENV
touch Pipfile.lock
- restore_cache:
keys:
- pip-packages-v1-{{ .Branch }}-{{ checksum "Pipfile.lock" }}
- pip-packages-v1-{{ .Branch }}-
- pip-packages-v1
- run:
name: Install requirements
command: |
sudo apt-get update
sudo apt-get install rsync -y
pip install -U pip
pip install -U setuptools wheel
pip install -r /home/circleci/<< pipeline.parameters.qa_repo_name >>/requirements.txt
cd /home/circleci/project/<< pipeline.parameters.app_core_name >>
hatch build
pip install dist/vizro*.tar.gz
- save_cache:
paths:
- ~/.venv/
key: pip-packages-v1-{{ .Branch }}-{{ checksum "Pipfile.lock" }}
echo $created_workflow_status
jobs:
run_tests:
docker:
- image: cimg/python:3.8.17
steps:
- checkout
- custom_checkout
- create_env_and_install_reqs
- chromedriver_install
- run:
name: Run dashboards and tests
command: |
export PYTHONPATH="/home/circleci/<< pipeline.parameters.qa_repo_name >>:$PYTHONPATH"
cd /home/circleci/<< pipeline.parameters.qa_repo_name >>
python dashboards/run_all_dashboards.py
dashboards/wait_for_dashboard.sh
TEST_FILES=$(circleci tests glob "tests/test_*.py" | circleci tests split --split-by=timings)
python -m pytest -v --reruns 1 --junitxml=test-results/junit.xml --override-ini junit_family=xunit1 $TEST_FILES
- run:
name: Collecting failed screenshots
command: |
mkdir /home/circleci/<< pipeline.parameters.qa_repo_name >>/failed_screenshots/
cd /home/circleci/<< pipeline.parameters.qa_repo_name >>/
cp tests*.png failed_screenshots
when: on_fail
- store_test_results:
path: /home/circleci/<< pipeline.parameters.qa_repo_name >>/test-results
- store_artifacts:
path: /home/circleci/<< pipeline.parameters.qa_repo_name >>/failed_screenshots/
if [[ "$created_workflow_status" != "success" ]]; then
echo "Workflow not successful - ${created_workflow_status}"
(exit -1)
fi
run_notebooks:
docker:
- image: cimg/python:3.8.17
steps:
- checkout
- custom_checkout
- create_env_and_install_reqs
- chromedriver_install
- run:
name: Run notebooks
command: |
export PYTHONPATH="/home/circleci/<< pipeline.parameters.qa_repo_name >>:$PYTHONPATH"
ipython kernel install --name "<< pipeline.parameters.qa_repo_name >>" --user
cd /home/circleci/project/<< pipeline.parameters.app_core_name >>
../../<< pipeline.parameters.qa_repo_name >>/scripts/run_notebooks.sh
- run:
name: Run notebooks as python scripts
command: |
export PYTHONPATH="/home/circleci/<< pipeline.parameters.qa_repo_name >>:$PYTHONPATH"
cd /home/circleci/project/<< pipeline.parameters.app_core_name >>
../../<< pipeline.parameters.qa_repo_name >>/scripts/convert_notebooks_to_python.sh
cd examples/jupyter
python app.py &
cd /home/circleci/<< pipeline.parameters.qa_repo_name >>
dashboards/wait_for_notebooks.sh
python -m pytest -v --reruns 1 --junitxml=test-results/junit.xml tests_notebooks
- run:
name: Collecting failed notebooks
command: |
mkdir /home/circleci/<< pipeline.parameters.qa_repo_name >>/failed_notebooks/
cd /home/circleci/project/<< pipeline.parameters.app_core_name >>
cp examples*.txt /home/circleci/<< pipeline.parameters.qa_repo_name >>/failed_notebooks/
when: on_fail
- run:
name: Collecting failed screenshots
command: |
mkdir /home/circleci/<< pipeline.parameters.qa_repo_name >>/failed_screenshots/
cd /home/circleci/<< pipeline.parameters.qa_repo_name >>/
cp tests*.png failed_screenshots
when: on_fail
- store_test_results:
path: /home/circleci/<< pipeline.parameters.qa_repo_name >>/test-results
- store_test_results:
path: /home/circleci/<< pipeline.parameters.qa_repo_name >>/failed_notebooks/
- store_artifacts:
path: /home/circleci/<< pipeline.parameters.qa_repo_name >>/failed_screenshots/
echo "Created workflow successful"
workflows:
version: 2
vizro-core:
jobs:
- run_tests:
post-steps:
- approve-job-in-triggering-pipeline
- trigger-new-pipeline:
context:
- circleci-api
- wait-for-triggered-pipeline:
type: approval
requires:
- trigger-new-pipeline
- check-status-of-triggered-pipeline:
requires:
- wait-for-triggered-pipeline
context:
- circleci-apix
- run_notebooks
- circleci-api
42 changes: 42 additions & 0 deletions vizro-core/changelog.d/20230926_135042_alexey_snigir_circleci_pipeline_change.md
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,42 @@
<!--
A new scriv changelog fragment.
Uncomment the section that is right (remove the HTML comment wrapper).
-->

<!--
### Removed
- A bullet item for the Removed category.
-->
<!--
### Added
- A bullet item for the Added category.
-->
<!--
### Changed
- A bullet item for the Changed category.
-->
<!--
### Deprecated
- A bullet item for the Deprecated category.
-->
<!--
### Fixed
- A bullet item for the Fixed category.
-->
<!--
### Security
- A bullet item for the Security category.
-->

0 comments on commit 4303ae0

Please sign in to comment.