forked from rucio/rucio
-
Notifications
You must be signed in to change notification settings - Fork 0
156 lines (152 loc) · 7.47 KB
/
imagecache.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
name: Daily image cache build for autotests
on:
workflow_dispatch: {}
schedule:
- cron: '0 3 * * 1-5'
jobs:
setup:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
- name: Identify branches
id: branches
run: |
BRANCHES="$(echo "$GITHUB_API_URL/repos/$GITHUB_REPOSITORY/branches{/branch}" | \
./tools/github/workflow/grabrelease.py --all --json --add master)"
echo "Will build image cache for branches: ${BRANCHES}"
echo "branches=${BRANCHES}" >> $GITHUB_OUTPUT
outputs:
branches: ${{ steps.branches.outputs.branches }}
build_autotests:
runs-on: ubuntu-latest
needs:
- setup
strategy:
fail-fast: false
matrix:
branch: ${{ fromJson(needs.setup.outputs.branches) }}
steps:
- uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
with:
ref: ${{ matrix.branch }}
- name: Check and get files
id: files
shell: bash
run: |
MATRIX_PARSER_PY="${{ github.workspace }}/tools/test/matrix_parser.py"
MATRIX_CONF="${{ github.workspace }}/etc/docker/test/matrix.yml"
BUILD_IMAGES_PY="${{ github.workspace }}/tools/test/build_images.py"
echo "requirements_met=$(if [[ -r "$MATRIX_CONF" && -x "$MATRIX_PARSER_PY" && -x "$BUILD_IMAGES_PY" ]]; then echo "true"; else echo "false"; fi)" >> $GITHUB_OUTPUT
echo "matrix_parser=$MATRIX_PARSER_PY" >> $GITHUB_OUTPUT
echo "matrix_configuration=$MATRIX_CONF" >> $GITHUB_OUTPUT
echo "build_images=$BUILD_IMAGES_PY" >> $GITHUB_OUTPUT
- name: Update pip, install python requirements for matrix parser
if: ${{ steps.files.outputs.requirements_met == 'true' }}
run: |
python3 -m pip install -U pip setuptools
python3 -m pip install -U PyYAML
- name: Identify Matrix
if: ${{ steps.files.outputs.requirements_met == 'true' }}
id: matrix
run: echo "matrix=$(${{ steps.files.outputs.matrix_parser }} < ${{ steps.files.outputs.matrix_configuration }})" >> $GITHUB_OUTPUT
- name: Build and upload images
if: ${{ steps.files.outputs.requirements_met == 'true' }}
id: images
shell: bash
run: |
BUILD_ARGS=("--output" "list" "--cache-repo" "ghcr.io/${{ github.repository }}" "--push-cache")
if [[ "${{ github.event_name }}" == "schedule" ]]; then
# only build without cache on schedule (to update distro packages)
BUILD_ARGS=("${BUILD_ARGS[@]}" "--build-no-cache")
fi
# the {} input for build_images.py is necessary until version 2
if echo '{}' | "${{ steps.files.outputs.build_images }}" --version-test 2; then
# new --branch option not available in older build_images.py revisions
BUILD_ARGS=("${BUILD_ARGS[@]}" "--branch" "${{ matrix.branch }}")
fi
if [[ "${{ matrix.branch }}" != "master" ]]; then
# The image tag is the same for the different rucio versions. This if ensures, that only the master gets builded. rucio#5572
echo "::warning::Not building image for branch ${{ matrix.branch }}, because it would overwrite master"
exit 0
fi
docker login https://ghcr.io -u ${{ github.actor }} -p ${{ secrets.GITHUB_TOKEN }}
i=0; until [ "$i" -ge 3 ]; do
IMAGES="$(echo '${{ steps.matrix.outputs.matrix }}' | "${{ steps.files.outputs.build_images }}" ${BUILD_ARGS[@]} ./etc/docker/test || echo "")"
if [[ -n "$IMAGES" ]]; then break;
else
i=$((i+1)); sleep 5;
echo "::warning::Building images failed, retrying…"
fi
done
docker logout https://ghcr.io
if [[ -z "$IMAGES" ]]; then echo "::error::Building images failed ultimately"; exit 1; fi
build_integration_tests:
runs-on: ubuntu-latest
needs:
- setup
strategy:
fail-fast: false
matrix:
branch: ${{ fromJson(needs.setup.outputs.branches) }}
steps:
- name: Checkout rucio containers repository
uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
with:
repository: rucio/containers
- name: Checkout rucio source
uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
with:
path: dev/rucio
ref: ${{ matrix.branch }}
- name: Check and get files
id: files
shell: bash
run: |
MATRIX_PARSER_PY="${{ github.workspace }}/dev/rucio/tools/test/matrix_parser.py"
MATRIX_CONF="${{ github.workspace }}/dev/rucio/etc/docker/test/matrix_integration_tests.yml"
BUILD_IMAGES_PY="${{ github.workspace }}/dev/rucio/tools/test/build_images.py"
echo "requirements_met=$(if [[ -r "$MATRIX_CONF" && -x "$MATRIX_PARSER_PY" && -x "$BUILD_IMAGES_PY" ]]; then echo "true"; else echo "false"; fi)" >> $GITHUB_OUTPUT
echo "matrix_parser=$MATRIX_PARSER_PY" >> $GITHUB_OUTPUT
echo "matrix_configuration=$MATRIX_CONF" >> $GITHUB_OUTPUT
echo "build_images=$BUILD_IMAGES_PY" >> $GITHUB_OUTPUT
- name: Update pip, install python requirements for matrix parser
if: ${{ steps.files.outputs.requirements_met == 'true' }}
run: |
python3 -m pip install -U pip setuptools
python3 -m pip install -U PyYAML
- name: Identify Matrix
if: ${{ steps.files.outputs.requirements_met == 'true' }}
id: matrix
run: echo "matrix=$(${{ steps.files.outputs.matrix_parser }} < ${{ steps.files.outputs.matrix_configuration }})" >> $GITHUB_OUTPUT
- name: Build and upload images
if: ${{ steps.files.outputs.requirements_met == 'true' }}
id: images
shell: bash
run: |
BUILD_ARGS=("--output" "list" "--cache-repo" "ghcr.io/${{ github.repository }}" "--push-cache")
if [[ "${{ github.event_name }}" == "schedule" ]]; then
# only build without cache on schedule (to update distro packages)
BUILD_ARGS=("${BUILD_ARGS[@]}" "--build-no-cache")
fi
# the {} input for build_images.py is necessary until version 2
if echo '{}' | "${{ steps.files.outputs.build_images }}" --version-test 2; then
# new --branch option not available in older build_images.py revisions
BUILD_ARGS=("${BUILD_ARGS[@]}" "--branch" "${{ matrix.branch }}")
fi
if [[ "${{ matrix.branch }}" != "master" ]]; then
# The image tag is the same for the different rucio versions. This if ensures, that only the master gets builded. rucio#5572
echo "::warning::Not building image for branch ${{ matrix.branch }}, because it would overwrite master"
exit 0
fi
docker login https://ghcr.io -u ${{ github.actor }} -p ${{ secrets.GITHUB_TOKEN }}
i=0; until [ "$i" -ge 3 ]; do
IMAGES="$(echo '${{ steps.matrix.outputs.matrix }}' | "${{ steps.files.outputs.build_images }}" ${BUILD_ARGS[@]} \
${{ github.workspace }}/dev || echo "")"
if [[ -n "$IMAGES" ]]; then break;
else
i=$((i+1)); sleep 5;
echo "::warning::Building images failed, retrying…"
fi
done
docker logout https://ghcr.io
if [[ -z "$IMAGES" ]]; then echo "::error::Building images failed ultimately"; exit 1; fi