-
Notifications
You must be signed in to change notification settings - Fork 117
402 lines (392 loc) · 13.7 KB
/
precommit.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
name: precommit test
on:
push:
branches: [ main ]
pull_request:
branches: '**'
workflow_dispatch:
inputs:
logLevel:
default: warning
description: "Log level"
required: true
tags:
description: "Test scenario tags"
concurrency:
group: ${{ github.ref }}
cancel-in-progress: true
jobs:
lint:
name: Check linting
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- name: Set up Python
uses: actions/setup-python@v2
with:
python-version: '3.8'
- name: Display Python version
run: python -c "import sys; import os; print(\"\n\".join(os.environ[\"PATH\"].split(os.pathsep))); print(sys.version); print(sys.executable);"
- name: Upgrade setuptools and pip
run: python -m pip install -U setuptools pip
- name: Install tox
run: python -m pip install tox
- name: Run fix_lint
run: python -m tox -e fix_lint
build:
needs: lint
name: Build Wheel File
runs-on: ubuntu-latest
steps:
- name: Checkout Code
uses: actions/checkout@v2
- name: Setup Python
uses: actions/setup-python@v2
with:
python-version: 3.8
- name: Upgrade setuptools and pip
run: python -m pip install -U setuptools pip wheel
- name: Generate wheel
run: python -m pip wheel -v -w dist --no-deps .
- name: Show wheels generated
run: ls -lh dist
- uses: actions/upload-artifact@v1
with:
name: wheel
path: dist/
test:
name: Test ${{ matrix.os.download_name }}-${{ matrix.python-version }}-${{ matrix.cloud-provider }}
needs: build
runs-on: ${{ matrix.os.image_name }}
strategy:
fail-fast: false
matrix:
os:
- image_name: ubuntu-latest
download_name: linux
- image_name: macos-latest
download_name: macos
- image_name: windows-latest
download_name: windows
python-version: ["3.8", "3.9", "3.10", "3.11"]
cloud-provider: [aws, azure, gcp]
steps:
- name: Checkout Code
uses: actions/checkout@v2
- name: Set up Python
uses: actions/setup-python@v2
with:
python-version: ${{ matrix.python-version }}
- name: Display Python version
run: python -c "import sys; print(sys.version)"
- name: Decrypt parameters.py
shell: bash
run: .github/scripts/decrypt_parameters.sh
env:
PARAMETER_PASSWORD: ${{ secrets.PARAMETER_PASSWORD }}
CLOUD_PROVIDER: ${{ matrix.cloud-provider }}
- name: Download wheel(s)
uses: actions/download-artifact@v2
with:
name: wheel
path: dist
- name: Show wheels downloaded
run: ls -lh dist
shell: bash
- name: Upgrade setuptools, pip and wheel
run: python -m pip install -U setuptools pip wheel
- name: Install tox
run: python -m pip install tox
- if: ${{ matrix.python-version != '3.11' && contains('macos', matrix.os.download_name) }}
name: Run doctests
run: python -m tox -e "py${PYTHON_VERSION}-doctest-notudf-ci"
env:
PYTHON_VERSION: ${{ matrix.python-version }}
cloud_provider: ${{ matrix.cloud-provider }}
PYTEST_ADDOPTS: --color=yes --tb=short
TOX_PARALLEL_NO_SPINNER: 1
# Specify SNOWFLAKE_IS_PYTHON_RUNTIME_TEST: 1 when adding >= python3.11 with no server-side support
# For example, see https://github.com/snowflakedb/snowpark-python/pull/681
shell: bash
- if: ${{ matrix.python-version != '3.11' }}
name: Run tests (excluding doctests)
run: python -m tox -e "py${PYTHON_VERSION/\./}-notdoctest-ci"
env:
PYTHON_VERSION: ${{ matrix.python-version }}
cloud_provider: ${{ matrix.cloud-provider }}
PYTEST_ADDOPTS: --color=yes --tb=short
TOX_PARALLEL_NO_SPINNER: 1
shell: bash
- if: ${{ matrix.python-version == '3.11' }}
name: Run tests (excluding udf and doctest tests)
run: python -m tox -e "py${PYTHON_VERSION/\./}-notudfdoctest"
env:
PYTHON_VERSION: ${{ matrix.python-version }}
cloud_provider: ${{ matrix.cloud-provider }}
PYTEST_ADDOPTS: --color=yes --tb=short
TOX_PARALLEL_NO_SPINNER: 1
SNOWFLAKE_IS_PYTHON_RUNTIME_TEST: 1
shell: bash
- name: Combine coverages
run: python -m tox -e coverage --skip-missing-interpreters false
shell: bash
env:
SNOWFLAKE_IS_PYTHON_RUNTIME_TEST: 1
- uses: actions/upload-artifact@v2
with:
name: coverage_${{ matrix.os.download_name }}-${{ matrix.python-version }}-${{ matrix.cloud-provider }}
path: |
.tox/.coverage
.tox/coverage.xml
test-fips:
name: Test FIPS linux-3.8-${{ matrix.cloud-provider }}
needs: build
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
cloud-provider: [aws]
steps:
- name: Checkout Code
uses: actions/checkout@v2
- name: Decrypt parameters.py
shell: bash
run: .github/scripts/decrypt_parameters.sh
env:
PARAMETER_PASSWORD: ${{ secrets.PARAMETER_PASSWORD }}
CLOUD_PROVIDER: ${{ matrix.cloud-provider }}
- name: Download wheel(s)
uses: actions/download-artifact@v2
with:
name: wheel
path: dist
- name: Show wheels downloaded
run: ls -lh dist
shell: bash
- name: Run tests
run: ./ci/test_fips_docker.sh
env:
PYTHON_VERSION: 3.8
cloud_provider: ${{ matrix.cloud-provider }}
PYTEST_ADDOPTS: --color=yes --tb=short
TOX_PARALLEL_NO_SPINNER: 1
shell: bash
- uses: actions/upload-artifact@v2
with:
name: coverage_linux-fips-3.8-${{ matrix.cloud-provider }}
path: |
.coverage
coverage.xml
test-disable-sql-simplifier: # Will be removed after sql simplifier is stable and no option to opt out.
name: Test Disable SQL Simplifier ${{ matrix.os.download_name }}-${{ matrix.python-version }}-${{ matrix.cloud-provider }}
needs: build
runs-on: ${{ matrix.os.image_name }}
strategy:
fail-fast: false
matrix:
os:
- image_name: macos-latest
download_name: macos # it includes doctest
python-version: ["3.8", "3.9", "3.10", "3.11"]
cloud-provider: [aws]
steps:
- name: Checkout Code
uses: actions/checkout@v2
- name: Set up Python
uses: actions/setup-python@v2
with:
python-version: ${{ matrix.python-version }}
- name: Display Python version
run: python -c "import sys; print(sys.version)"
- name: Decrypt parameters.py
shell: bash
run: .github/scripts/decrypt_parameters.sh
env:
PARAMETER_PASSWORD: ${{ secrets.PARAMETER_PASSWORD }}
CLOUD_PROVIDER: ${{ matrix.cloud-provider }}
- name: Download wheel(s)
uses: actions/download-artifact@v2
with:
name: wheel
path: dist
- name: Show wheels downloaded
run: ls -lh dist
shell: bash
- name: Upgrade setuptools, pip and wheel
run: python -m pip install -U setuptools pip wheel
- name: Install tox
run: python -m pip install tox
- if: ${{ matrix.python-version != '3.11' && contains('macos', matrix.os.download_name) }}
name: Run doctests
run: python -m tox -e "py${PYTHON_VERSION}-doctest-notudf-ci"
env:
PYTHON_VERSION: ${{ matrix.python-version }}
cloud_provider: ${{ matrix.cloud-provider }}
PYTEST_ADDOPTS: --color=yes --tb=short --disable_sql_simplifier
TOX_PARALLEL_NO_SPINNER: 1
shell: bash
- if: ${{ matrix.python-version != '3.11' }}
name: Run tests (excluding doctests)
run: python -m tox -e "py${PYTHON_VERSION/\./}-notdoctest-ci"
env:
PYTHON_VERSION: ${{ matrix.python-version }}
cloud_provider: ${{ matrix.cloud-provider }}
PYTEST_ADDOPTS: --color=yes --tb=short --disable_sql_simplifier
TOX_PARALLEL_NO_SPINNER: 1
shell: bash
- if: ${{ matrix.python-version == '3.11' }}
name: Run tests (excluding udf and doctest tests)
run: python -m tox -e "py${PYTHON_VERSION/\./}-notudfdoctest"
env:
PYTHON_VERSION: ${{ matrix.python-version }}
cloud_provider: ${{ matrix.cloud-provider }}
PYTEST_ADDOPTS: --color=yes --tb=short --disable_sql_simplifier
TOX_PARALLEL_NO_SPINNER: 1
SNOWFLAKE_IS_PYTHON_RUNTIME_TEST: 1
shell: bash
- name: Combine coverages
run: python -m tox -e coverage --skip-missing-interpreters false
shell: bash
env:
SNOWFLAKE_IS_PYTHON_RUNTIME_TEST: 1
- uses: actions/upload-artifact@v2
with:
name: coverage_${{ matrix.os.download_name }}-${{ matrix.python-version }}-${{ matrix.cloud-provider }}-disable-sql-simplifier
path: |
.tox/.coverage
.tox/coverage.xml
test-no-pandas:
name: Test when pandas is not installed ${{ matrix.os.download_name }}-${{ matrix.python-version }}-${{ matrix.cloud-provider }}
needs: build
runs-on: ${{ matrix.os.image_name }}
strategy:
fail-fast: false
matrix:
os:
- image_name: macos-latest
download_name: macos
python-version: ["3.8"]
cloud-provider: [aws]
steps:
- name: Checkout Code
uses: actions/checkout@v2
- name: Set up Python
uses: actions/setup-python@v2
with:
python-version: ${{ matrix.python-version }}
- name: Display Python version
run: python -c "import sys; print(sys.version)"
- name: Decrypt parameters.py
shell: bash
run: .github/scripts/decrypt_parameters.sh
env:
PARAMETER_PASSWORD: ${{ secrets.PARAMETER_PASSWORD }}
CLOUD_PROVIDER: ${{ matrix.cloud-provider }}
- name: Download wheel(s)
uses: actions/download-artifact@v2
with:
name: wheel
path: dist
- name: Show wheels downloaded
run: ls -lh dist
shell: bash
- name: Upgrade setuptools, pip and wheel
run: python -m pip install -U setuptools pip wheel
- name: Install tox
run: python -m pip install tox
- name: Run all tests
run: python -m tox -e "nopandas"
env:
PYTHON_VERSION: ${{ matrix.python-version }}
cloud_provider: ${{ matrix.cloud-provider }}
PYTEST_ADDOPTS: --color=yes --tb=short
TOX_PARALLEL_NO_SPINNER: 1
# Specify SNOWFLAKE_IS_PYTHON_RUNTIME_TEST: 1 when adding >= python3.11 with no server-side support
# For example, see https://github.com/snowflakedb/snowpark-python/pull/681
shell: bash
- name: Combine coverages
run: python -m tox -e coverage --skip-missing-interpreters false
shell: bash
env:
SNOWFLAKE_IS_PYTHON_RUNTIME_TEST: 1
- uses: actions/upload-artifact@v2
with:
name: coverage_nopandas
path: |
.tox/.coverage
.tox/coverage.xml
combine-coverage:
if: ${{ success() || failure() }}
name: Combine coverage
needs: test
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- uses: actions/download-artifact@v2
with:
path: artifacts
- name: Set up Python
uses: actions/setup-python@v2
with:
python-version: '3.8'
- name: Display Python version
run: python -c "import sys; print(sys.version)"
- name: Upgrade setuptools and pip
run: python -m pip install -U setuptools pip
- name: Install tox
run: python -m pip install tox
- name: Collect all coverages to one dir
run: |
python -c '
from pathlib import Path
import shutil
src_dir = Path("artifacts")
dst_dir = Path(".") / ".tox"
dst_dir.mkdir()
for src_file in src_dir.glob("*/.coverage"):
dst_file = dst_dir / ".coverage.{}".format(src_file.parent.name[9:])
print("{} copy to {}".format(src_file, dst_file))
shutil.copy(str(src_file), str(dst_file))'
- name: Combine coverages
run: python -m tox -e coverage
- name: Publish html coverage
uses: actions/upload-artifact@v2
with:
name: overall_cov_html
path: .tox/htmlcov
- name: Publish xml coverage
uses: actions/upload-artifact@v2
with:
name: overall_cov_xml
path: .tox/coverage.xml
- uses: codecov/codecov-action@v1
with:
file: .tox/coverage.xml
doc:
needs: lint
name: Build Doc
runs-on: ubuntu-latest
steps:
- name: Checkout Code
uses: actions/checkout@v2
- name: Setup Python
uses: actions/setup-python@v2
with:
python-version: '3.8'
- name: Upgrade setuptools and pip
run: python -m pip install -U setuptools pip
- name: Install Snowpark
run: python -m pip install ".[development, pandas]"
- name: Install Sphinx
run: python -m pip install sphinx
- name: Build document
working-directory: docs
# treat warning as failure but complete the entire process
run: |
make clean
make html SPHINXOPTS="-W --keep-going"
- name: Upload html files
uses: actions/upload-artifact@v2
with:
name: docs
path: docs/build/html