Skip to content

Commit

Permalink
add
Browse files Browse the repository at this point in the history
  • Loading branch information
sfc-gh-jdu committed Jun 5, 2024
1 parent 28fdb9d commit 69207c3
Show file tree
Hide file tree
Showing 4 changed files with 69 additions and 29 deletions.
68 changes: 68 additions & 0 deletions .github/workflows/daily_modin_precommit.yml
Original file line number Diff line number Diff line change
Expand Up @@ -215,6 +215,74 @@ jobs:
.tox/.coverage
.tox/coverage.xml
test-enable-cte-optimization:
name: Test Enable CTE Optimization modin-${{ matrix.os.download_name }}-${{ matrix.python-version }}-${{ matrix.cloud-provider }}
needs: build
runs-on: ${{ matrix.os.image_name }}
strategy:
fail-fast: false
matrix:
os:
- image_name: macos-latest
download_name: macos # it includes doctest
python-version: ["3.9", "3.10", "3.11"]
cloud-provider: [aws]
steps:
- name: Checkout Code
uses: actions/checkout@v4
- name: Set up Python
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}
- name: Display Python version
run: python -c "import sys; print(sys.version)"
- name: Decrypt parameters.py
shell: bash
run: .github/scripts/decrypt_parameters.sh
env:
PARAMETER_PASSWORD: ${{ secrets.PARAMETER_PASSWORD }}
CLOUD_PROVIDER: ${{ matrix.cloud-provider }}
- name: Download wheel(s)
uses: actions/download-artifact@v3
with:
name: wheel
path: dist
- name: Show wheels downloaded
run: ls -lh dist
shell: bash
- name: Upgrade setuptools, pip and wheel
run: python -m pip install -U setuptools pip wheel
- name: Install tox
run: python -m pip install tox
- if: ${{ contains('macos', matrix.os.download_name) }}
name: Run Snowpark pandas API doctests
run: python -m tox -e "py${PYTHON_VERSION}-doctest-snowparkpandasdoctest-modin-ci"
env:
PYTHON_VERSION: ${{ matrix.python-version }}
cloud_provider: ${{ matrix.cloud-provider }}
PYTEST_ADDOPTS: --color=yes --tb=short --enable_cte_optimization
TOX_PARALLEL_NO_SPINNER: 1
shell: bash
- name: Run Snowpark pandas API tests (excluding doctests)
run: python -m tox -e "py${PYTHON_VERSION/\./}-snowparkpandasdailynotdoctest-modin-ci"
env:
PYTHON_VERSION: ${{ matrix.python-version }}
cloud_provider: ${{ matrix.cloud-provider }}
PYTEST_ADDOPTS: --color=yes --tb=short --enable_cte_optimization --skip_sql_count_check --ignore=tests/integ/modin/test_sql_counter.py
TOX_PARALLEL_NO_SPINNER: 1
shell: bash
- name: Combine coverages
run: python -m tox -e coverage --skip-missing-interpreters false
shell: bash
env:
SNOWFLAKE_IS_PYTHON_RUNTIME_TEST: 1
- uses: actions/upload-artifact@v4
with:
name: coverage_${{ matrix.os.download_name }}-${{ matrix.python-version }}-${{ matrix.cloud-provider }}-enable-cte-optimization
path: |
.tox/.coverage
.tox/coverage.xml
combine-coverage:
if: ${{ success() || failure() }}
name: Combine coverage
Expand Down
20 changes: 0 additions & 20 deletions src/snowflake/snowpark/_internal/analyzer/cte_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -60,11 +60,6 @@ def traverse(root: "TreeNode") -> None:
while len(current_level) > 0:
next_level = []
for node in current_level:
# all subqueries under dynamic pivot node will not be optimized now
# due to a server side bug
# TODO: SNOW-1413967 Remove it when the bug is fixed
if is_dynamic_pivot_node(node):
continue
node_count_map[node] += 1
for child in node.children_plan_nodes:
# converting non-SELECT child query to SELECT query here,
Expand All @@ -89,21 +84,6 @@ def is_duplicate_subtree(node: "TreeNode") -> bool:
return True
return False

def is_dynamic_pivot_node(node: "TreeNode") -> bool:
from snowflake.snowpark._internal.analyzer.select_statement import (
SelectSnowflakePlan,
)
from snowflake.snowpark._internal.analyzer.snowflake_plan import SnowflakePlan
from snowflake.snowpark._internal.analyzer.unary_plan_node import Pivot

if isinstance(node, SelectSnowflakePlan):
source_plan = node.snowflake_plan.source_plan
elif isinstance(node, SnowflakePlan):
source_plan = node.source_plan
else:
return False
return isinstance(source_plan, Pivot) and source_plan.pivot_values is None

traverse(root)
return {node for node in node_count_map if is_duplicate_subtree(node)}

Expand Down
3 changes: 1 addition & 2 deletions tests/integ/modin/frame/test_quantile.py
Original file line number Diff line number Diff line change
Expand Up @@ -131,9 +131,8 @@ def test_quantile_datetime_negative():
snow_df.quantile(numeric_only=False)


@pytest.mark.xfail(
@pytest.mark.skip(
reason="Bug in quantile emitting large amount of queries except for small data. TODO: SNOW-1229442",
strict=True,
)
@sql_count_checker(query_count=0)
def test_quantile_large():
Expand Down
7 changes: 0 additions & 7 deletions tests/integ/test_cte.py
Original file line number Diff line number Diff line change
Expand Up @@ -422,13 +422,6 @@ def test_pivot_unpivot(session):
check_result(session, df_result, expect_cte_optimized=True)
assert count_number_of_ctes(df_result.queries["queries"][-1]) == 1

# Because of SNOW-1375062, dynamic pivot doesn't work with nested CTE
# TODO: SNOW-1413967 Remove it when the bug is fixed
df_nested = session.table("monthly_sales").select("*")
df_nested = df_nested.union_all(df_nested).select("*")
df_dynamic_pivot = df_nested.pivot("month").sum("amount")
check_result(session, df_dynamic_pivot, expect_cte_optimized=False)


def test_window_function(session):
window1 = (
Expand Down

0 comments on commit 69207c3

Please sign in to comment.