Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

SNOW-1432019 Calculate subtree query complexity #1657

Merged
merged 41 commits into from
Jun 14, 2024
Merged
Show file tree
Hide file tree
Changes from 35 commits
Commits
Show all changes
41 commits
Select commit Hold shift + click to select a range
e0ff34d
Calculate query complexity
sfc-gh-aalam May 22, 2024
7f714e3
make subtree computation iterative; telemetry
sfc-gh-aalam May 22, 2024
b279769
compute complexity for expressions
sfc-gh-aalam May 29, 2024
4582bb2
add tests
sfc-gh-aalam May 29, 2024
7abd06c
tests passing
sfc-gh-aalam May 30, 2024
1e3a34c
fix test
sfc-gh-aalam May 30, 2024
aad17ee
Merge branch 'main' into aalam-SNOW-1432019-calculate-candidacy-scores
sfc-gh-aalam May 30, 2024
46d8b9f
fix typing issues
sfc-gh-aalam May 30, 2024
d38b3a4
Merge branch 'aalam-SNOW-1432019-calculate-candidacy-scores' of githu…
sfc-gh-aalam May 30, 2024
88c7e7c
use new approach
sfc-gh-aalam Jun 4, 2024
fb68aa0
fix type checks
sfc-gh-aalam Jun 4, 2024
8da7ffd
fix async job test
sfc-gh-aalam Jun 4, 2024
8474b5e
fix async job test
sfc-gh-aalam Jun 4, 2024
f72ce8c
remove change added in error
sfc-gh-aalam Jun 5, 2024
310fbd2
remove change added in error
sfc-gh-aalam Jun 5, 2024
e79f277
add description on async fix
sfc-gh-aalam Jun 5, 2024
d5451ce
move Counter typing into complexity stat
sfc-gh-aalam Jun 5, 2024
3b639bb
rename file
sfc-gh-aalam Jun 5, 2024
909ed67
address feedback
sfc-gh-aalam Jun 5, 2024
ed83869
fix bad imports
sfc-gh-aalam Jun 5, 2024
026e428
fix type hints
sfc-gh-aalam Jun 5, 2024
df9bfac
rename
sfc-gh-aalam Jun 5, 2024
ced5d78
rename file
sfc-gh-aalam Jun 5, 2024
995f107
refactor
sfc-gh-aalam Jun 5, 2024
2b2f8a5
fix lint and type hints
sfc-gh-aalam Jun 5, 2024
7f636e6
fix classification for Interval expression
sfc-gh-aalam Jun 5, 2024
e40129d
added some unit tests
sfc-gh-aalam Jun 6, 2024
8607aa4
add unit test
sfc-gh-aalam Jun 6, 2024
3830820
fix typing
sfc-gh-aalam Jun 6, 2024
68855ac
align with doc
sfc-gh-aalam Jun 6, 2024
40fc65d
Merge branch 'main' into aalam-SNOW-1432019-calculate-candidacy-scores
sfc-gh-aalam Jun 7, 2024
1299ee9
fix SelectSQL
sfc-gh-aalam Jun 12, 2024
4a901f4
rename to node_complexity; add setter for cumulative complexity expre…
sfc-gh-aalam Jun 12, 2024
a555e1d
use Dict type hint instead of Counter
sfc-gh-aalam Jun 12, 2024
3008878
rename dict add function
sfc-gh-aalam Jun 12, 2024
4bd8ce6
fix type hints using enums and do not count alias twice
sfc-gh-aalam Jun 13, 2024
9035a70
align complexity stat calculation
sfc-gh-aalam Jun 13, 2024
b12b7f6
fix telemetry test
sfc-gh-aalam Jun 13, 2024
3f6c997
update comment
sfc-gh-aalam Jun 13, 2024
0c1c9da
merge with main
sfc-gh-aalam Jun 13, 2024
8a03068
fix unit test
sfc-gh-aalam Jun 14, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion src/snowflake/snowpark/_internal/analyzer/analyzer.py
Original file line number Diff line number Diff line change
Expand Up @@ -927,7 +927,7 @@ def do_resolve_with_resolved_children(
)

if isinstance(logical_plan, UnresolvedRelation):
return self.plan_builder.table(logical_plan.name)
return self.plan_builder.table(logical_plan.name, logical_plan)
sfc-gh-yzou marked this conversation as resolved.
Show resolved Hide resolved

if isinstance(logical_plan, SnowflakeCreateTable):
return self.plan_builder.save_as_table(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,9 @@
Expression,
derive_dependent_columns,
)
from snowflake.snowpark._internal.analyzer.query_plan_analysis_utils import (
PlanNodeCategory,
)


class BinaryExpression(Expression):
Expand All @@ -26,6 +29,10 @@ def __str__(self):
def dependent_column_names(self) -> Optional[AbstractSet[str]]:
return derive_dependent_columns(self.left, self.right)

@property
def plan_node_category(self) -> PlanNodeCategory:
return PlanNodeCategory.LOW_IMPACT


class BinaryArithmeticExpression(BinaryExpression):
pass
Expand Down
39 changes: 37 additions & 2 deletions src/snowflake/snowpark/_internal/analyzer/binary_plan_node.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,13 @@
# Copyright (c) 2012-2024 Snowflake Computing Inc. All rights reserved.
#

from typing import List, Optional
from typing import Dict, List, Optional

from snowflake.snowpark._internal.analyzer.expression import Expression
from snowflake.snowpark._internal.analyzer.query_plan_analysis_utils import (
PlanNodeCategory,
sum_node_complexities,
)
from snowflake.snowpark._internal.analyzer.snowflake_plan_node import LogicalPlan
from snowflake.snowpark._internal.error_message import SnowparkClientExceptionMessages

Expand Down Expand Up @@ -69,7 +73,10 @@ def __init__(self, left: LogicalPlan, right: LogicalPlan) -> None:


class SetOperation(BinaryNode):
pass
@property
def plan_node_category(self) -> PlanNodeCategory:
# (left) operator (right)
return PlanNodeCategory.SET_OPERATION


class Except(SetOperation):
Expand Down Expand Up @@ -187,3 +194,31 @@ def __init__(
@property
def sql(self) -> str:
return self.join_type.sql

@property
def plan_node_category(self) -> PlanNodeCategory:
return PlanNodeCategory.JOIN

@property
def individual_node_complexity(self) -> Dict[str, int]:
# SELECT * FROM (left) AS left_alias join_type_sql JOIN (right) AS right_alias match_cond, using_cond, join_cond
score = {self.plan_node_category.value: 1}
if isinstance(self.join_type, UsingJoin) and self.join_type.using_columns:
score = sum_node_complexities(
score,
{PlanNodeCategory.COLUMN.value: len(self.join_type.using_columns)},
)
score = (
sum_node_complexities(score, self.join_condition.cumulative_node_complexity)
if self.join_condition
else score
)

score = (
sum_node_complexities(
score, self.match_condition.cumulative_node_complexity
)
if self.match_condition
else score
)
return score
Loading
Loading