Cleanup CI clusters #38
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Cleanup CI clusters | |
| on: | |
| workflow_dispatch: | |
| schedule: | |
| - cron: '0 21 * * *' # Run at 9PM - image sync runs at midnight | |
| jobs: | |
| ci_cleanup: | |
| name: ci-cleanup | |
| concurrency: ${{ github.workflow }}-${{ github.ref }}-${{ matrix.cloud }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| cloud: | |
| - LEAFCLOUD | |
| - SMS | |
| - ARCUS | |
| runs-on: ubuntu-22.04 | |
| env: | |
| OS_CLOUD: openstack | |
| CI_CLOUD: ${{ matrix.cloud }} | |
| steps: | |
| - uses: actions/checkout@v2 | |
| - name: Record which cloud CI is running on | |
| run: | | |
| echo CI_CLOUD: ${{ env.CI_CLOUD }} | |
| - name: Setup environment | |
| run: | | |
| python3 -m venv venv | |
| . venv/bin/activate | |
| pip install -U pip | |
| pip install $(grep -o 'python-openstackclient[><=0-9\.]*' requirements.txt) | |
| shell: bash | |
| - name: Write clouds.yaml | |
| run: | | |
| mkdir -p ~/.config/openstack/ | |
| echo "${{ secrets[format('{0}_CLOUDS_YAML', env.CI_CLOUD)] }}" > ~/.config/openstack/clouds.yaml | |
| shell: bash | |
| - name: Find CI clusters | |
| run: | | |
| . venv/bin/activate | |
| CI_CLUSTERS=$(openstack server list | grep --only-matching 'slurmci-RL.-[0-9]\+' | sort | uniq || true) | |
| echo "DEBUG: Raw CI clusters: $CI_CLUSTERS" | |
| if [[ -z "$CI_CLUSTERS" ]]; then | |
| echo "No matching CI clusters found." | |
| else | |
| # Flatten multiline value so can be passed as env var | |
| CI_CLUSTERS_FORMATTED=$(echo "$CI_CLUSTERS" | tr '\n' ' ' | sed 's/ $//') | |
| echo "DEBUG: Formatted CI clusters: $CI_CLUSTERS_FORMATTED" | |
| echo "ci_clusters=$CI_CLUSTERS_FORMATTED" >> $GITHUB_ENV | |
| fi | |
| shell: bash | |
| - name: Delete clusters if control node not tagged with keep | |
| run: | | |
| . venv/bin/activate | |
| if [[ -z ${ci_clusters} ]]; then | |
| echo "No clusters to delete." | |
| exit 0 | |
| fi | |
| for cluster_prefix in ${ci_clusters} | |
| do | |
| echo "Processing cluster: $cluster_prefix" | |
| TAGS=$(openstack server show ${cluster_prefix}-control --column tags --format value) | |
| if [[ $TAGS =~ "keep" ]]; then | |
| echo "Skipping ${cluster_prefix} - control instance is tagged as keep" | |
| else | |
| ./dev/delete-cluster.py ${cluster_prefix} --force | |
| fi | |
| done | |
| shell: bash |