Skip to content

SNOW-841405 Fix df copy and enable basic diamond shaped joins for simplifier #4202

SNOW-841405 Fix df copy and enable basic diamond shaped joins for simplifier

SNOW-841405 Fix df copy and enable basic diamond shaped joins for simplifier #4202

Workflow file for this run

name: precommit test
on:
push:
branches: [ main ]
pull_request:
branches: '**'
workflow_dispatch:
inputs:
logLevel:
default: warning
description: "Log level"
required: true
tags:
description: "Test scenario tags"
concurrency:
group: ${{ github.ref }}
cancel-in-progress: true
jobs:
lint:
name: Check linting
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- name: Set up Python
uses: actions/setup-python@v2
with:
python-version: '3.8'
- name: Display Python version
run: python -c "import sys; import os; print(\"\n\".join(os.environ[\"PATH\"].split(os.pathsep))); print(sys.version); print(sys.executable);"
- name: Upgrade setuptools and pip
run: python -m pip install -U setuptools pip
- name: Install tox
run: python -m pip install tox
- name: Run fix_lint
run: python -m tox -e fix_lint
build:
needs: lint
name: Build Wheel File
runs-on: ubuntu-latest
steps:
- name: Checkout Code
uses: actions/checkout@v2
- name: Setup Python
uses: actions/setup-python@v2
with:
python-version: 3.8
- name: Upgrade setuptools and pip
run: python -m pip install -U setuptools pip wheel
- name: Generate wheel
run: python -m pip wheel -v -w dist --no-deps .
- name: Show wheels generated
run: ls -lh dist
- uses: actions/upload-artifact@v1
with:
name: wheel
path: dist/
test:
name: Test ${{ matrix.os.download_name }}-${{ matrix.python-version }}-${{ matrix.cloud-provider }}
needs: build
runs-on: ${{ matrix.os.image_name }}
strategy:
fail-fast: false
matrix:
os:
- image_name: ubuntu-latest
download_name: linux
- image_name: macos-latest
download_name: macos
- image_name: windows-latest
download_name: windows
python-version: ["3.8", "3.9", "3.10", "3.11"]
cloud-provider: [aws, azure, gcp]
steps:
- name: Checkout Code
uses: actions/checkout@v2
- name: Set up Python
uses: actions/setup-python@v2
with:
python-version: ${{ matrix.python-version }}
- name: Display Python version
run: python -c "import sys; print(sys.version)"
- name: Decrypt parameters.py
shell: bash
run: .github/scripts/decrypt_parameters.sh
env:
PARAMETER_PASSWORD: ${{ secrets.PARAMETER_PASSWORD }}
CLOUD_PROVIDER: ${{ matrix.cloud-provider }}
- name: Download wheel(s)
uses: actions/download-artifact@v2
with:
name: wheel
path: dist
- name: Show wheels downloaded
run: ls -lh dist
shell: bash
- name: Upgrade setuptools, pip and wheel
run: python -m pip install -U setuptools pip wheel
- name: Install tox
run: python -m pip install tox
- if: ${{ matrix.python-version != '3.11' && contains('macos', matrix.os.download_name) }}
name: Run doctests
run: python -m tox -e "py${PYTHON_VERSION}-doctest-notudf-ci"
env:
PYTHON_VERSION: ${{ matrix.python-version }}
cloud_provider: ${{ matrix.cloud-provider }}
PYTEST_ADDOPTS: --color=yes --tb=short
TOX_PARALLEL_NO_SPINNER: 1
# Specify SNOWFLAKE_IS_PYTHON_RUNTIME_TEST: 1 when adding >= python3.11 with no server-side support
# For example, see https://github.com/snowflakedb/snowpark-python/pull/681
shell: bash
- if: ${{ matrix.python-version != '3.11' }}
name: Run tests (excluding doctests)
run: python -m tox -e "py${PYTHON_VERSION/\./}-notdoctest-ci"
env:
PYTHON_VERSION: ${{ matrix.python-version }}
cloud_provider: ${{ matrix.cloud-provider }}
PYTEST_ADDOPTS: --color=yes --tb=short
TOX_PARALLEL_NO_SPINNER: 1
shell: bash
- if: ${{ matrix.python-version == '3.11' }}
name: Run tests (excluding udf and doctest tests)
run: python -m tox -e "py${PYTHON_VERSION/\./}-notudfdoctest"
env:
PYTHON_VERSION: ${{ matrix.python-version }}
cloud_provider: ${{ matrix.cloud-provider }}
PYTEST_ADDOPTS: --color=yes --tb=short
TOX_PARALLEL_NO_SPINNER: 1
SNOWFLAKE_IS_PYTHON_RUNTIME_TEST: 1
shell: bash
- name: Combine coverages
run: python -m tox -e coverage --skip-missing-interpreters false
shell: bash
env:
SNOWFLAKE_IS_PYTHON_RUNTIME_TEST: 1
- uses: actions/upload-artifact@v2
with:
name: coverage_${{ matrix.os.download_name }}-${{ matrix.python-version }}-${{ matrix.cloud-provider }}
path: |
.tox/.coverage
.tox/coverage.xml
test-fips:
name: Test FIPS linux-3.8-${{ matrix.cloud-provider }}
needs: build
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
cloud-provider: [aws]
steps:
- name: Checkout Code
uses: actions/checkout@v2
- name: Decrypt parameters.py
shell: bash
run: .github/scripts/decrypt_parameters.sh
env:
PARAMETER_PASSWORD: ${{ secrets.PARAMETER_PASSWORD }}
CLOUD_PROVIDER: ${{ matrix.cloud-provider }}
- name: Download wheel(s)
uses: actions/download-artifact@v2
with:
name: wheel
path: dist
- name: Show wheels downloaded
run: ls -lh dist
shell: bash
- name: Run tests
run: ./ci/test_fips_docker.sh
env:
PYTHON_VERSION: 3.8
cloud_provider: ${{ matrix.cloud-provider }}
PYTEST_ADDOPTS: --color=yes --tb=short
TOX_PARALLEL_NO_SPINNER: 1
shell: bash
- uses: actions/upload-artifact@v2
with:
name: coverage_linux-fips-3.8-${{ matrix.cloud-provider }}
path: |
.coverage
coverage.xml
test-disable-sql-simplifier: # Will be removed after sql simplifier is stable and no option to opt out.
name: Test Disable SQL Simplifier ${{ matrix.os.download_name }}-${{ matrix.python-version }}-${{ matrix.cloud-provider }}
needs: build
runs-on: ${{ matrix.os.image_name }}
strategy:
fail-fast: false
matrix:
os:
- image_name: macos-latest
download_name: macos # it includes doctest
python-version: ["3.8", "3.9", "3.10", "3.11"]
cloud-provider: [aws]
steps:
- name: Checkout Code
uses: actions/checkout@v2
- name: Set up Python
uses: actions/setup-python@v2
with:
python-version: ${{ matrix.python-version }}
- name: Display Python version
run: python -c "import sys; print(sys.version)"
- name: Decrypt parameters.py
shell: bash
run: .github/scripts/decrypt_parameters.sh
env:
PARAMETER_PASSWORD: ${{ secrets.PARAMETER_PASSWORD }}
CLOUD_PROVIDER: ${{ matrix.cloud-provider }}
- name: Download wheel(s)
uses: actions/download-artifact@v2
with:
name: wheel
path: dist
- name: Show wheels downloaded
run: ls -lh dist
shell: bash
- name: Upgrade setuptools, pip and wheel
run: python -m pip install -U setuptools pip wheel
- name: Install tox
run: python -m pip install tox
- if: ${{ matrix.python-version != '3.11' && contains('macos', matrix.os.download_name) }}
name: Run doctests
run: python -m tox -e "py${PYTHON_VERSION}-doctest-notudf-ci"
env:
PYTHON_VERSION: ${{ matrix.python-version }}
cloud_provider: ${{ matrix.cloud-provider }}
PYTEST_ADDOPTS: --color=yes --tb=short --disable_sql_simplifier
TOX_PARALLEL_NO_SPINNER: 1
shell: bash
- if: ${{ matrix.python-version != '3.11' }}
name: Run tests (excluding doctests)
run: python -m tox -e "py${PYTHON_VERSION/\./}-notdoctest-ci"
env:
PYTHON_VERSION: ${{ matrix.python-version }}
cloud_provider: ${{ matrix.cloud-provider }}
PYTEST_ADDOPTS: --color=yes --tb=short --disable_sql_simplifier
TOX_PARALLEL_NO_SPINNER: 1
shell: bash
- if: ${{ matrix.python-version == '3.11' }}
name: Run tests (excluding udf and doctest tests)
run: python -m tox -e "py${PYTHON_VERSION/\./}-notudfdoctest"
env:
PYTHON_VERSION: ${{ matrix.python-version }}
cloud_provider: ${{ matrix.cloud-provider }}
PYTEST_ADDOPTS: --color=yes --tb=short --disable_sql_simplifier
TOX_PARALLEL_NO_SPINNER: 1
SNOWFLAKE_IS_PYTHON_RUNTIME_TEST: 1
shell: bash
- name: Combine coverages
run: python -m tox -e coverage --skip-missing-interpreters false
shell: bash
env:
SNOWFLAKE_IS_PYTHON_RUNTIME_TEST: 1
- uses: actions/upload-artifact@v2
with:
name: coverage_${{ matrix.os.download_name }}-${{ matrix.python-version }}-${{ matrix.cloud-provider }}-disable-sql-simplifier
path: |
.tox/.coverage
.tox/coverage.xml
test-no-pandas:
name: Test when pandas is not installed ${{ matrix.os.download_name }}-${{ matrix.python-version }}-${{ matrix.cloud-provider }}
needs: build
runs-on: ${{ matrix.os.image_name }}
strategy:
fail-fast: false
matrix:
os:
- image_name: macos-latest
download_name: macos
python-version: ["3.8"]
cloud-provider: [aws]
steps:
- name: Checkout Code
uses: actions/checkout@v2
- name: Set up Python
uses: actions/setup-python@v2
with:
python-version: ${{ matrix.python-version }}
- name: Display Python version
run: python -c "import sys; print(sys.version)"
- name: Decrypt parameters.py
shell: bash
run: .github/scripts/decrypt_parameters.sh
env:
PARAMETER_PASSWORD: ${{ secrets.PARAMETER_PASSWORD }}
CLOUD_PROVIDER: ${{ matrix.cloud-provider }}
- name: Download wheel(s)
uses: actions/download-artifact@v2
with:
name: wheel
path: dist
- name: Show wheels downloaded
run: ls -lh dist
shell: bash
- name: Upgrade setuptools, pip and wheel
run: python -m pip install -U setuptools pip wheel
- name: Install tox
run: python -m pip install tox
- name: Run all tests
run: python -m tox -e "nopandas"
env:
PYTHON_VERSION: ${{ matrix.python-version }}
cloud_provider: ${{ matrix.cloud-provider }}
PYTEST_ADDOPTS: --color=yes --tb=short
TOX_PARALLEL_NO_SPINNER: 1
# Specify SNOWFLAKE_IS_PYTHON_RUNTIME_TEST: 1 when adding >= python3.11 with no server-side support
# For example, see https://github.com/snowflakedb/snowpark-python/pull/681
shell: bash
- name: Combine coverages
run: python -m tox -e coverage --skip-missing-interpreters false
shell: bash
env:
SNOWFLAKE_IS_PYTHON_RUNTIME_TEST: 1
- uses: actions/upload-artifact@v2
with:
name: coverage_nopandas
path: |
.tox/.coverage
.tox/coverage.xml
combine-coverage:
if: ${{ success() || failure() }}
name: Combine coverage
needs: test
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- uses: actions/download-artifact@v2
with:
path: artifacts
- name: Set up Python
uses: actions/setup-python@v2
with:
python-version: '3.8'
- name: Display Python version
run: python -c "import sys; print(sys.version)"
- name: Upgrade setuptools and pip
run: python -m pip install -U setuptools pip
- name: Install tox
run: python -m pip install tox
- name: Collect all coverages to one dir
run: |
python -c '
from pathlib import Path
import shutil
src_dir = Path("artifacts")
dst_dir = Path(".") / ".tox"
dst_dir.mkdir()
for src_file in src_dir.glob("*/.coverage"):
dst_file = dst_dir / ".coverage.{}".format(src_file.parent.name[9:])
print("{} copy to {}".format(src_file, dst_file))
shutil.copy(str(src_file), str(dst_file))'
- name: Combine coverages
run: python -m tox -e coverage
- name: Publish html coverage
uses: actions/upload-artifact@v2
with:
name: overall_cov_html
path: .tox/htmlcov
- name: Publish xml coverage
uses: actions/upload-artifact@v2
with:
name: overall_cov_xml
path: .tox/coverage.xml
- uses: codecov/codecov-action@v1
with:
file: .tox/coverage.xml
doc:
needs: lint
name: Build Doc
runs-on: ubuntu-latest
steps:
- name: Checkout Code
uses: actions/checkout@v2
- name: Setup Python
uses: actions/setup-python@v2
with:
python-version: '3.8'
- name: Upgrade setuptools and pip
run: python -m pip install -U setuptools pip
- name: Install Snowpark
run: python -m pip install ".[development, pandas]"
- name: Install Sphinx
run: python -m pip install sphinx
- name: Build document
working-directory: docs
# treat warning as failure but complete the entire process
run: |
make clean
make html SPHINXOPTS="-W --keep-going"
- name: Upload html files
uses: actions/upload-artifact@v2
with:
name: docs
path: docs/build/html