SNOW-1418523: concurrent file operations #10822
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: precommit test | |
| on: | |
| push: | |
| branches: [ main ] | |
| pull_request: | |
| branches: '**' | |
| workflow_dispatch: | |
| inputs: | |
| logLevel: | |
| default: warning | |
| description: "Log level" | |
| required: true | |
| tags: | |
| description: "Test scenario tags" | |
| concurrency: | |
| group: ${{ github.ref }} | |
| cancel-in-progress: true | |
| jobs: | |
| lint: | |
| name: Check linting | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set up Python | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: '3.9' | |
| - name: Display Python version | |
| run: python -c "import sys; import os; print(\"\n\".join(os.environ[\"PATH\"].split(os.pathsep))); print(sys.version); print(sys.executable);" | |
| - name: Upgrade setuptools and pip | |
| run: python -m pip install -U setuptools pip | |
| - name: Install tox | |
| run: python -m pip install tox | |
| - name: Run fix_lint | |
| run: python -m tox -e fix_lint | |
| type_checking: | |
| name: Type Checking | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set up Python | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: '3.9' | |
| - name: Display Python version | |
| run: python -c "import sys; import os; print(\"\n\".join(os.environ[\"PATH\"].split(os.pathsep))); print(sys.version); print(sys.executable);" | |
| - name: Upgrade setuptools and pip | |
| run: python -m pip install -U setuptools pip | |
| - name: Install tox | |
| run: python -m pip install tox | |
| - name: Run pyright on Selected Files | |
| run: python -m tox -e pyright | |
| build: | |
| needs: lint | |
| name: Build Wheel File | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout Code | |
| uses: actions/checkout@v4 | |
| - name: Setup Python | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: 3.9 | |
| - name: Upgrade setuptools and pip | |
| run: python -m pip install -U setuptools pip wheel | |
| - name: Generate wheel | |
| run: python -m pip wheel -v -w dist --no-deps . | |
| - name: Show wheels generated | |
| run: ls -lh dist | |
| - uses: actions/upload-artifact@v4 | |
| with: | |
| include-hidden-files: true | |
| name: wheel | |
| path: dist/ | |
| test: | |
| name: Test py-${{ matrix.os }}-${{ matrix.python-version }}-${{ matrix.cloud-provider }} | |
| needs: build | |
| runs-on: ${{ matrix.os }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| os: [macos-latest, windows-latest-64-cores, ubuntu-latest-64-cores] | |
| python-version: ["3.9", "3.10", "3.11"] | |
| cloud-provider: [aws, gcp, azure] | |
| exclude: | |
| # only run macos with aws py3.9 for doctest | |
| - os: macos-latest | |
| python-version: "3.10" | |
| - os: macos-latest | |
| python-version: "3.11" | |
| - os: macos-latest | |
| python-version: "3.9" | |
| cloud-provider: azure | |
| - os: macos-latest | |
| python-version: "3.9" | |
| cloud-provider: gcp | |
| # only run ubuntu with py3.9 on aws and py3.10 on azure | |
| - os: ubuntu-latest-64-cores | |
| python-version: "3.11" | |
| - os: ubuntu-latest-64-cores | |
| python-version: "3.9" | |
| cloud-provider: azure | |
| - os: ubuntu-latest-64-cores | |
| python-version: "3.9" | |
| cloud-provider: gcp | |
| - os: ubuntu-latest-64-cores | |
| python-version: "3.10" | |
| cloud-provider: aws | |
| - os: ubuntu-latest-64-cores | |
| python-version: "3.10" | |
| cloud-provider: gcp | |
| # only run windows with py3.10 on gcp | |
| - os: windows-latest-64-cores | |
| python-version: "3.9" | |
| - os: windows-latest-64-cores | |
| python-version: "3.10" | |
| - os: windows-latest-64-cores | |
| python-version: "3.11" | |
| cloud-provider: aws | |
| - os: windows-latest-64-cores | |
| python-version: "3.11" | |
| cloud-provider: azure | |
| steps: | |
| - name: Checkout Code | |
| uses: actions/checkout@v4 | |
| - name: Set up Python | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Display Python version | |
| run: python -c "import sys; print(sys.version)" | |
| - name: Decrypt parameters.py | |
| shell: bash | |
| run: .github/scripts/decrypt_parameters.sh | |
| env: | |
| PARAMETER_PASSWORD: ${{ secrets.PARAMETER_PASSWORD }} | |
| CLOUD_PROVIDER: ${{ matrix.cloud-provider }} | |
| - name: Download wheel(s) | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: wheel | |
| path: dist | |
| - name: Show wheels downloaded | |
| run: ls -lh dist | |
| shell: bash | |
| - name: Upgrade setuptools, pip and wheel | |
| run: python -m pip install -U setuptools pip wheel | |
| - name: Install tox | |
| run: python -m pip install tox | |
| # we only run doctest on macos | |
| - if: ${{ matrix.os == 'macos-latest' }} | |
| name: Run doctests | |
| run: python -m tox -e "py${PYTHON_VERSION}-doctest-notudf-ci" | |
| env: | |
| PYTHON_VERSION: ${{ matrix.python-version }} | |
| cloud_provider: ${{ matrix.cloud-provider }} | |
| PYTEST_ADDOPTS: --color=yes --tb=short | |
| TOX_PARALLEL_NO_SPINNER: 1 | |
| # Specify SNOWFLAKE_IS_PYTHON_RUNTIME_TEST: 1 when adding >= python3.12 with no server-side support | |
| # For example, see https://github.com/snowflakedb/snowpark-python/pull/681 | |
| shell: bash | |
| # do not run other tests for macos | |
| - if: ${{ matrix.os != 'macos-latest' }} | |
| name: Run tests (excluding doctests) | |
| run: python -m tox -e "py${PYTHON_VERSION/\./}-notdoctest-ci" | |
| env: | |
| PYTHON_VERSION: ${{ matrix.python-version }} | |
| cloud_provider: ${{ matrix.cloud-provider }} | |
| PYTEST_ADDOPTS: --color=yes --tb=short | |
| TOX_PARALLEL_NO_SPINNER: 1 | |
| shell: bash | |
| - name: Combine coverages | |
| run: python -m tox -e coverage --skip-missing-interpreters false | |
| shell: bash | |
| env: | |
| SNOWFLAKE_IS_PYTHON_RUNTIME_TEST: 1 | |
| - uses: actions/upload-artifact@v4 | |
| with: | |
| include-hidden-files: true | |
| name: coverage_${{ matrix.os }}-${{ matrix.python-version }}-${{ matrix.cloud-provider }} | |
| path: | | |
| .tox/.coverage | |
| .tox/coverage.xml | |
| test-local-testing: | |
| name: Test Local Testing Module py-${{ matrix.os }}-${{ matrix.python-version }} | |
| needs: build | |
| runs-on: ${{ matrix.os }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| os: [macos-latest, windows-latest, ubuntu-latest] | |
| python-version: [ "3.8", "3.9", "3.10", "3.11" ] | |
| cloud-provider: [aws] | |
| steps: | |
| - name: Checkout Code | |
| uses: actions/checkout@v4 | |
| - name: Set up Python | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Display Python version | |
| run: python -c "import sys; print(sys.version)" | |
| - name: Decrypt parameters.py | |
| shell: bash | |
| run: .github/scripts/decrypt_parameters.sh | |
| env: | |
| PARAMETER_PASSWORD: ${{ secrets.PARAMETER_PASSWORD }} | |
| CLOUD_PROVIDER: ${{ matrix.cloud-provider }} | |
| - name: Download wheel(s) | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: wheel | |
| path: dist | |
| - name: Show wheels downloaded | |
| run: ls -lh dist | |
| shell: bash | |
| - name: Upgrade setuptools, pip and wheel | |
| run: python -m pip install -U setuptools pip wheel | |
| - name: Install tox | |
| run: python -m pip install tox | |
| - name: Run tests | |
| run: python -m tox -e "py${PYTHON_VERSION/\./}-local" | |
| env: | |
| PYTHON_VERSION: ${{ matrix.python-version }} | |
| cloud_provider: ${{ matrix.cloud-provider }} | |
| PYTEST_ADDOPTS: --color=yes --tb=short | |
| TOX_PARALLEL_NO_SPINNER: 1 | |
| SNOWFLAKE_IS_PYTHON_RUNTIME_TEST: 1 | |
| shell: bash | |
| - name: Combine coverages | |
| run: python -m tox -e coverage --skip-missing-interpreters false | |
| shell: bash | |
| env: | |
| SNOWFLAKE_IS_PYTHON_RUNTIME_TEST: 1 | |
| - uses: actions/upload-artifact@v4 | |
| with: | |
| include-hidden-files: true | |
| name: coverage_${{ matrix.os }}-${{ matrix.python-version }}-local-testing | |
| path: | | |
| .tox/.coverage | |
| .tox/coverage.xml | |
| test-enable-cte-optimization: | |
| name: Test Enable CTE Optimization py-${{ matrix.os }}-${{ matrix.python-version }}-${{ matrix.cloud-provider }} | |
| needs: build | |
| runs-on: ${{ matrix.os }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| os: [ubuntu-latest-64-cores] | |
| python-version: ["3.9"] | |
| cloud-provider: [aws] | |
| steps: | |
| - name: Checkout Code | |
| uses: actions/checkout@v4 | |
| - name: Set up Python | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Display Python version | |
| run: python -c "import sys; print(sys.version)" | |
| - name: Decrypt parameters.py | |
| shell: bash | |
| run: .github/scripts/decrypt_parameters.sh | |
| env: | |
| PARAMETER_PASSWORD: ${{ secrets.PARAMETER_PASSWORD }} | |
| CLOUD_PROVIDER: ${{ matrix.cloud-provider }} | |
| - name: Download wheel(s) | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: wheel | |
| path: dist | |
| - name: Show wheels downloaded | |
| run: ls -lh dist | |
| shell: bash | |
| - name: Upgrade setuptools, pip and wheel | |
| run: python -m pip install -U setuptools pip wheel | |
| - name: Install tox | |
| run: python -m pip install tox | |
| - name: Run tests (excluding doctests) | |
| run: python -m tox -e "py${PYTHON_VERSION/\./}-notdoctest-ci" | |
| env: | |
| PYTHON_VERSION: ${{ matrix.python-version }} | |
| cloud_provider: ${{ matrix.cloud-provider }} | |
| PYTEST_ADDOPTS: --color=yes --tb=short --enable_cte_optimization | |
| TOX_PARALLEL_NO_SPINNER: 1 | |
| shell: bash | |
| - name: Combine coverages | |
| run: python -m tox -e coverage --skip-missing-interpreters false | |
| shell: bash | |
| env: | |
| SNOWFLAKE_IS_PYTHON_RUNTIME_TEST: 1 | |
| - uses: actions/upload-artifact@v4 | |
| with: | |
| include-hidden-files: true | |
| name: coverage_${{ matrix.os }}-${{ matrix.python-version }}-${{ matrix.cloud-provider }}-enable-cte-optimization | |
| path: | | |
| .tox/.coverage | |
| .tox/coverage.xml | |
| test-snowpark-pandas: | |
| name: Test modin-${{ matrix.os }}-${{ matrix.python-version }}-${{ matrix.cloud-provider }} | |
| needs: build | |
| runs-on: ${{ matrix.os }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| os: [macos-latest, windows-latest-64-cores, ubuntu-latest-64-cores] | |
| python-version: [ "3.9", "3.10", "3.11" ] | |
| cloud-provider: [aws, gcp, azure] | |
| exclude: | |
| # only run macos with aws py3.9 for doctest | |
| - os: macos-latest | |
| python-version: "3.10" | |
| - os: macos-latest | |
| python-version: "3.11" | |
| - os: macos-latest | |
| python-version: "3.9" | |
| cloud-provider: azure | |
| - os: macos-latest | |
| python-version: "3.9" | |
| cloud-provider: gcp | |
| # only run ubuntu with py3.9 on aws and py3.10 on azure | |
| - os: ubuntu-latest-64-cores | |
| python-version: "3.11" | |
| - os: ubuntu-latest-64-cores | |
| python-version: "3.9" | |
| cloud-provider: azure | |
| - os: ubuntu-latest-64-cores | |
| python-version: "3.9" | |
| cloud-provider: gcp | |
| - os: ubuntu-latest-64-cores | |
| python-version: "3.10" | |
| cloud-provider: aws | |
| - os: ubuntu-latest-64-cores | |
| python-version: "3.10" | |
| cloud-provider: gcp | |
| # only run windows with py3.10 on gcp | |
| - os: windows-latest-64-cores | |
| python-version: "3.9" | |
| - os: windows-latest-64-cores | |
| python-version: "3.10" | |
| - os: windows-latest-64-cores | |
| python-version: "3.11" | |
| cloud-provider: aws | |
| - os: windows-latest-64-cores | |
| python-version: "3.11" | |
| cloud-provider: azure | |
| steps: | |
| - name: Checkout Code | |
| uses: actions/checkout@v4 | |
| - name: Set up Python | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Display Python version | |
| run: python -c "import sys; print(sys.version)" | |
| - name: Decrypt parameters.py | |
| shell: bash | |
| run: .github/scripts/decrypt_parameters.sh | |
| env: | |
| PARAMETER_PASSWORD: ${{ secrets.PARAMETER_PASSWORD }} | |
| CLOUD_PROVIDER: ${{ matrix.cloud-provider }} | |
| - name: Download wheel(s) | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: wheel | |
| path: dist | |
| - name: Show wheels downloaded | |
| run: ls -lh dist | |
| shell: bash | |
| - name: Upgrade setuptools, pip and wheel | |
| run: python -m pip install -U setuptools pip wheel | |
| - name: Install tox | |
| run: python -m pip install tox | |
| # only run doctest for macos on aws | |
| - if: ${{ matrix.os == 'macos-latest' && matrix.cloud-provider == 'aws' }} | |
| name: Run Snowpark pandas API doctests | |
| run: python -m tox -e "py${PYTHON_VERSION}-doctest-snowparkpandasdoctest-modin-ci" | |
| env: | |
| PYTHON_VERSION: ${{ matrix.python-version }} | |
| cloud_provider: ${{ matrix.cloud-provider }} | |
| PYTEST_ADDOPTS: --color=yes --tb=short | |
| TOX_PARALLEL_NO_SPINNER: 1 | |
| # Specify SNOWFLAKE_IS_PYTHON_RUNTIME_TEST: 1 when adding >= python3.11 with no server-side support | |
| # For example, see https://github.com/snowflakedb/snowpark-python/pull/681 | |
| shell: bash | |
| # do not run other tests for macos on aws | |
| - if: ${{ !(matrix.os == 'macos-latest' && matrix.cloud-provider == 'aws') }} | |
| name: Run Snowpark pandas API tests (excluding doctests) | |
| run: python -m tox -e "py${PYTHON_VERSION/\./}-snowparkpandasnotdoctest-modin-ci" | |
| env: | |
| PYTHON_VERSION: ${{ matrix.python-version }} | |
| cloud_provider: ${{ matrix.cloud-provider }} | |
| PYTEST_ADDOPTS: --color=yes --tb=short | |
| TOX_PARALLEL_NO_SPINNER: 1 | |
| shell: bash | |
| - name: Combine coverages | |
| run: python -m tox -e coverage --skip-missing-interpreters false | |
| shell: bash | |
| env: | |
| SNOWFLAKE_IS_PYTHON_RUNTIME_TEST: 1 | |
| - uses: actions/upload-artifact@v4 | |
| with: | |
| include-hidden-files: true | |
| name: coverage_${{ matrix.os }}-${{ matrix.python-version }}-${{ matrix.cloud-provider }}-snowpark-pandas-testing | |
| path: | | |
| .tox/.coverage | |
| .tox/coverage.xml | |
| test-modin-extra-without-pandas-extra: | |
| name: Test modin extra without pandas extra py-${{ matrix.os.download_name }}-${{ matrix.python-version }}-${{ matrix.cloud-provider }}. | |
| needs: build | |
| runs-on: ${{ matrix.os.image_name }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| os: | |
| - image_name: macos-latest | |
| download_name: macos | |
| python-version: ["3.9"] | |
| cloud-provider: [aws] | |
| steps: | |
| - name: Checkout Code | |
| uses: actions/checkout@v4 | |
| - name: Set up Python | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Display Python version | |
| run: python -c "import sys; print(sys.version)" | |
| - name: Decrypt parameters.py | |
| shell: bash | |
| run: .github/scripts/decrypt_parameters.sh | |
| env: | |
| PARAMETER_PASSWORD: ${{ secrets.PARAMETER_PASSWORD }} | |
| CLOUD_PROVIDER: ${{ matrix.cloud-provider }} | |
| - name: Download wheel(s) | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: wheel | |
| path: dist | |
| - name: Show wheels downloaded | |
| run: ls -lh dist | |
| shell: bash | |
| - name: Upgrade setuptools, pip and wheel | |
| run: python -m pip install -U setuptools pip wheel | |
| - name: Install tox | |
| run: python -m pip install tox | |
| - name: Run all tests | |
| run: python -m tox -e "modin_extra_without_pandas_extra" | |
| env: | |
| PYTHON_VERSION: ${{ matrix.python-version }} | |
| cloud_provider: ${{ matrix.cloud-provider }} | |
| PYTEST_ADDOPTS: --color=yes --tb=short | |
| TOX_PARALLEL_NO_SPINNER: 1 | |
| shell: bash | |
| - name: Combine coverages | |
| run: python -m tox -e coverage --skip-missing-interpreters false | |
| shell: bash | |
| - uses: actions/upload-artifact@v4 | |
| with: | |
| include-hidden-files: true | |
| name: coverage_modin_extra_without_pandas_extra | |
| path: | | |
| .tox/.coverage | |
| .tox/coverage.xml | |
| combine-coverage: | |
| if: ${{ success() || failure() }} | |
| name: Combine coverage | |
| needs: | |
| - test | |
| - test-local-testing | |
| - test-enable-cte-optimization | |
| - test-snowpark-pandas | |
| - test-modin-extra-without-pandas-extra | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout Code | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - uses: actions/download-artifact@v4 | |
| with: | |
| path: artifacts | |
| - name: Set up Python | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: '3.9' | |
| - name: Display Python version | |
| run: python -c "import sys; print(sys.version)" | |
| - name: Upgrade setuptools and pip | |
| run: python -m pip install -U setuptools pip | |
| - name: Install tox | |
| run: python -m pip install tox | |
| - name: Collect all coverages to one dir | |
| run: | | |
| python -c ' | |
| from pathlib import Path | |
| import shutil | |
| src_dir = Path("artifacts") | |
| dst_dir = Path(".") / ".tox" | |
| dst_dir.mkdir() | |
| for src_file in src_dir.glob("*/.coverage"): | |
| dst_file = dst_dir / ".coverage.{}".format(src_file.parent.name[9:]) | |
| print("{} copy to {}".format(src_file, dst_file)) | |
| shutil.copy(str(src_file), str(dst_file))' | |
| - name: Combine coverages | |
| run: python -m tox -e coverage | |
| - name: Publish html coverage | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| include-hidden-files: true | |
| name: overall_cov_html | |
| path: .tox/htmlcov | |
| - name: Publish xml coverage | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| include-hidden-files: true | |
| name: overall_cov_xml | |
| path: .tox/coverage.xml | |
| - uses: codecov/codecov-action@v4 | |
| with: | |
| file: .tox/coverage.xml | |
| token: ${{ secrets.CODECOV_TOKEN }} | |
| - name: Show coverage diff | |
| run: | | |
| pip install diff_cover | |
| git log -n 5 --pretty=oneline | |
| git fetch -f origin main:main || true | |
| diff-cover .tox/coverage.xml --compare-branch=main --fail-under=95 | |
| doc: | |
| needs: lint | |
| name: Build Doc | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout Code | |
| uses: actions/checkout@v4 | |
| - name: Setup Python | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: '3.9' | |
| - name: Upgrade setuptools and pip | |
| run: python -m pip install -U setuptools pip | |
| - name: Install Snowpark | |
| run: python -m pip install ".[modin-development]" | |
| - name: Install Sphinx | |
| run: python -m pip install sphinx | |
| - name: Build document | |
| working-directory: docs | |
| # treat warning as failure but complete the entire process | |
| run: | | |
| make clean | |
| make html SPHINXOPTS="-W --keep-going" | |
| - name: Upload html files | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| include-hidden-files: true | |
| name: docs | |
| path: docs/build/html |