-
Notifications
You must be signed in to change notification settings - Fork 187
146 lines (125 loc) · 5.07 KB
/
test_common.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
name: test common
on:
pull_request:
branches:
- master
- devel
workflow_dispatch:
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
env:
RUNTIME__LOG_LEVEL: ERROR
jobs:
get_docs_changes:
uses: ./.github/workflows/get_docs_changes.yml
run_common:
name: Tests common dlt code
needs: get_docs_changes
if: needs.get_docs_changes.outputs.changes_outside_docs == 'true'
strategy:
fail-fast: false
matrix:
os: ["ubuntu-latest", "macos-latest", "windows-latest"]
python-version: ["3.11.x"]
# Test all python versions on ubuntu only
include:
- python-version: "3.8.x"
os: "ubuntu-latest"
- python-version: "3.9.x"
os: "ubuntu-latest"
- python-version: "3.10.x"
os: "ubuntu-latest"
- python-version: "3.12.x"
os: "ubuntu-latest"
defaults:
run:
shell: bash
runs-on: ${{ matrix.os }}
steps:
- name: Check out
uses: actions/checkout@master
- name: Setup Python
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}
- name: Install Poetry
# https://github.com/snok/install-poetry#running-on-windows
uses: snok/[email protected]
with:
virtualenvs-create: true
virtualenvs-in-project: true
installer-parallel: true
# NOTE: do not cache. we want to have a clean state each run and we upgrade depdendencies later
# - name: Load cached venv
# id: cached-poetry-dependencies
# uses: actions/cache@v3
# with:
# # path: ${{ steps.pip-cache.outputs.dir }}
# path: .venv
# key: venv-${{ matrix.os }}-${{ matrix.python-version }}-${{ hashFiles('**/poetry.lock') }}
- name: Install dependencies
run: poetry install --no-interaction --with sentry-sdk
- run: |
poetry run pytest tests/common tests/normalize tests/reflection tests/sources tests/load/test_dummy_client.py tests/extract/test_extract.py tests/extract/test_sources.py tests/pipeline/test_pipeline_state.py
if: runner.os != 'Windows'
name: Run common tests with minimum dependencies Linux/MAC
- run: |
poetry run pytest tests/common tests/normalize tests/reflection tests/sources tests/load/test_dummy_client.py tests/extract/test_extract.py tests/extract/test_sources.py tests/pipeline/test_pipeline_state.py -m "not forked"
if: runner.os == 'Windows'
name: Run common tests with minimum dependencies Windows
shell: cmd
- name: Install duckdb dependencies
run: poetry install --no-interaction -E duckdb --with sentry-sdk
- run: |
poetry run pytest tests/pipeline/test_pipeline.py tests/pipeline/test_import_export_schema.py
if: runner.os != 'Windows'
name: Run pipeline smoke tests with minimum deps Linux/MAC
- run: |
poetry run pytest tests/pipeline/test_pipeline.py tests/pipeline/test_import_export_schema.py
if: runner.os == 'Windows'
name: Run smoke tests with minimum deps Windows
shell: cmd
- name: Install pyarrow
run: poetry install --no-interaction -E duckdb -E cli -E parquet --with sentry-sdk
- run: |
poetry run pytest tests/pipeline/test_pipeline_extra.py -k arrow
if: runner.os != 'Windows'
name: Run pipeline tests with pyarrow but no pandas installed
- run: |
poetry run pytest tests/pipeline/test_pipeline_extra.py -k arrow
if: runner.os == 'Windows'
name: Run pipeline tests with pyarrow but no pandas installed Windows
shell: cmd
- name: Install pipeline dependencies
run: poetry install --no-interaction -E duckdb -E cli -E parquet --with sentry-sdk --with pipeline
- run: |
poetry run pytest tests/extract tests/pipeline tests/libs tests/cli/common tests/destinations
if: runner.os != 'Windows'
name: Run extract and pipeline tests Linux/MAC
- run: |
poetry run pytest tests/extract tests/pipeline tests/libs tests/cli/common tests/destinations
if: runner.os == 'Windows'
name: Run extract tests Windows
shell: cmd
# - name: Install Pydantic 1.0
# run: pip install "pydantic<2"
# - run: |
# poetry run pytest tests/libs
# if: runner.os != 'Windows'
# name: Run extract and pipeline tests Linux/MAC
# - run: |
# poetry run pytest tests/libs
# if: runner.os == 'Windows'
# name: Run extract tests Windows
# shell: cmd
matrix_job_required_check:
name: Common tests
needs: run_common
runs-on: ubuntu-latest
if: always()
steps:
- name: Check matrix job results
if: contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled')
run: |
echo "One or more matrix job tests failed or were cancelled. You may need to re-run them." && exit 1