Use pytest mark to run subset of integ modin tests in Snowfort (#1703)

Signed-off-by: Labanya Mukhopadhyay <[email protected]>
Co-authored-by: azhan <[email protected]>

tests/integ/modin/frame/test_apply.py

@@ -603,6 +603,7 @@ def g(v):
 ]
 
 
+@pytest.mark.modin_sp_short_regress
 @pytest.mark.parametrize("data, apply_func", TRANSFORM_DATA_FUNC_MAP)
 @sql_count_checker(query_count=0)
 def test_basic_dataframe_transform(data, apply_func):

tests/integ/modin/frame/test_head_tail.py

@@ -29,6 +29,7 @@ def eval_result_and_query_with_no_join(
     assert_snowpark_pandas_equals_to_pandas_without_dtypecheck(snow, native, **kwargs)
 
 
+@pytest.mark.modin_sp_short_regress
 @pytest.mark.parametrize(
     "n",
     [1, None, 0, -1, -10, 5, 10],
@@ -50,6 +51,7 @@ def test_head_tail(n, default_index_snowpark_pandas_df, default_index_native_df)
     )
 
 
+@pytest.mark.modin_sp_short_regress
 @pytest.mark.parametrize(
     "n",
     [1, None, 0, -1, -10, 5, 10],

tests/integ/modin/frame/test_round.py

@@ -21,6 +21,7 @@
 zero_only_decimals = pytest.mark.parametrize("decimals", [0])
 
 
+@pytest.mark.modin_sp_precommit
 @all_decimals
 @sql_count_checker(query_count=1)
 def test_df_round(decimals):

tests/integ/modin/frame/test_sample.py

@@ -16,6 +16,7 @@ def ignore_index(request):
     return request.param
 
 
+@pytest.mark.modin_sp_precommit
 @sql_count_checker(query_count=1)
 def test_df_sample_cols():
     data = np.random.randint(100, size=(20, 20))

tests/integ/modin/groupby/test_groupby_basic_agg.py

@@ -58,6 +58,7 @@ def eval_groupby_result(
     return snowpark_pandas_groupby, pandas_groupby
 
 
+@pytest.mark.modin_sp_short_regress
 @pytest.mark.parametrize("by", ["col1", ["col3"], ["col5"]])
 @sql_count_checker(query_count=2)
 def test_basic_single_group_row_groupby(

tests/integ/modin/pivot/test_pivot_multiple.py

@@ -14,6 +14,7 @@
 
 
 @sql_count_checker(query_count=1, join_count=1)
+@pytest.mark.modin_sp_precommit
 def test_pivot_table_single_index_single_column_multiple_values(df_data):
     pivot_table_test_helper(
         df_data,

tests/integ/modin/pivot/test_pivot_single.py

@@ -29,6 +29,7 @@ def test_pivot_table_no_index_single_column_single_value(df_data):
     )
 
 
+@pytest.mark.modin_sp_short_regress
 @pytest.mark.parametrize(
     "aggfunc",
     [

tests/integ/modin/test_merge.py

@@ -57,6 +57,7 @@ def how(request):
     return request.param
 
 
+@pytest.mark.modin_sp_short_regress
 @sql_count_checker(query_count=2, join_count=2)
 def test_merge(left_df, right_df, how):
     res = pd.merge(left_df, right_df, on="A", how=how)

tests/integ/modin/tools/test_to_datetime.py

@@ -48,6 +48,7 @@ def utc(request):
 
 
 class TestTimeConversionFormats:
+    @pytest.mark.modin_sp_precommit
     @pytest.mark.parametrize("readonly", [True, False])
     @sql_count_checker(query_count=2)
     def test_to_datetime_readonly(self, readonly):

tests/unit/modin/test_internal_frame.py

@@ -37,6 +37,7 @@ def __init__(self, sp_df: OrderedDataFrame, internal: InternalFrame) -> None:
         self.internal_frame: InternalFrame = internal
 
 
+@pytest.mark.modin_sp_short_regress
 @pytest.fixture(scope="function")
 @patch("snowflake.snowpark.dataframe.DataFrame")
 def test_dataframes(mock_dataframe) -> TestDataFrames:
@@ -77,6 +78,7 @@ def test_dataframes(mock_dataframe) -> TestDataFrames:
     return TestDataFrames(ordered_dataframe, internal_frame)
 
 
+@pytest.mark.modin_sp_short_regress
 @pytest.fixture(scope="function")
 @patch("snowflake.snowpark.dataframe.DataFrame")
 def test_dataframes_with_multiindex_on_column(mock_dataframe) -> TestDataFrames:
@@ -113,6 +115,7 @@ def test_dataframes_with_multiindex_on_column(mock_dataframe) -> TestDataFrames:
     return TestDataFrames(ordered_dataframe, internal_frame)
 
 
+@pytest.mark.modin_sp_short_regress
 def test_snowflake_quoted_identifier_without_quote_negative(test_dataframes) -> None:
     with pytest.raises(AssertionError) as exc:
         InternalFrame.create(
@@ -127,6 +130,7 @@ def test_snowflake_quoted_identifier_without_quote_negative(test_dataframes) ->
     assert "Found not-quoted identifier for 'dataframe column':'a'" in str(exc.value)
 
 
+@pytest.mark.modin_sp_short_regress
 def test_column_labels_and_quoted_identifiers_have_same_length_negative(
     test_dataframes,
 ) -> None:
@@ -153,6 +157,7 @@ def test_column_labels_and_quoted_identifiers_have_same_length_negative(
         )
 
 
+@pytest.mark.modin_sp_short_regress
 def test_internal_frame_missing_data_column_negative(test_dataframes):
     with pytest.raises(AssertionError) as exc:
         InternalFrame.create(
@@ -169,6 +174,7 @@ def test_internal_frame_missing_data_column_negative(test_dataframes):
     )
 
 
+@pytest.mark.modin_sp_short_regress
 def test_internal_frame_missing_index_column_negative(test_dataframes):
     with pytest.raises(AssertionError) as exc:
         InternalFrame.create(
@@ -185,6 +191,7 @@ def test_internal_frame_missing_index_column_negative(test_dataframes):
     )
 
 
+@pytest.mark.modin_sp_short_regress
 def test_internal_frame_properties(test_dataframes) -> None:
     internal_frame = test_dataframes.internal_frame
     # check index_column_snowflake_quoted_identifiers
@@ -214,6 +221,7 @@ def test_internal_frame_properties(test_dataframes) -> None:
     assert ordering_column_snowflake_quoted_identifiers == ['"INDEX"']
 
 
+@pytest.mark.modin_sp_short_regress
 def test_pandas_label_as_empty_and_none(test_dataframes) -> None:
     internal_frame = InternalFrame.create(
         ordered_dataframe=test_dataframes.ordered_dataframe,
@@ -232,6 +240,7 @@ def test_pandas_label_as_empty_and_none(test_dataframes) -> None:
     ]
 
 
+@pytest.mark.modin_sp_short_regress
 def test_ordering_column_snowpark_column() -> None:
     ordering_column = OrderingColumn('"A"')
     snowpark_column = ordering_column.snowpark_column
@@ -262,6 +271,7 @@ def test_ordering_column_snowpark_column() -> None:
     assert isinstance(snowpark_column._expression.null_ordering, NullsFirst)
 
 
+@pytest.mark.modin_sp_short_regress
 def test_internal_frame_ordering_columns(test_dataframes) -> None:
     # ordering column is part of the index + data column
     test_dataframes.ordered_dataframe._ordering_columns_tuple = (
@@ -320,6 +330,7 @@ def test_internal_frame_ordering_columns(test_dataframes) -> None:
     ]
 
 
+@pytest.mark.modin_sp_short_regress
 @pytest.mark.parametrize("pandas_label", VALID_PANDAS_LABELS)
 def test_data_column_pandas_index_names(pandas_label, test_dataframes) -> None:
     test_dataframes.ordered_dataframe._ordering_columns_tuple_tuple = (
@@ -338,6 +349,7 @@ def test_data_column_pandas_index_names(pandas_label, test_dataframes) -> None:
     assert internal_frame.data_column_pandas_index_names == [pandas_label]
 
 
+@pytest.mark.modin_sp_short_regress
 def test_data_column_pandas_multiindex(
     test_dataframes_with_multiindex_on_column,
 ) -> None:
@@ -347,6 +359,7 @@ def test_data_column_pandas_multiindex(
     )
 
 
+@pytest.mark.modin_sp_short_regress
 @pytest.mark.parametrize(
     "data_column_pandas_labels, data_column_pandas_index_names, expected_error_message",
     [
@@ -407,6 +420,7 @@ def test_data_column_pandas_multiindex_negative(
         )
 
 
+@pytest.mark.modin_sp_short_regress
 def test_get_snowflake_quoted_identifiers_by_pandas_labels_empty(
     test_dataframes,
 ) -> None:
@@ -417,6 +431,7 @@ def test_get_snowflake_quoted_identifiers_by_pandas_labels_empty(
     )
 
 
+@pytest.mark.modin_sp_short_regress
 def test_get_snowflake_quoted_identifiers_by_pandas_labels_empty_not_include_index(
     test_dataframes,
 ) -> None:
@@ -439,6 +454,7 @@ def test_get_snowflake_quoted_identifiers_by_pandas_labels_empty_not_include_ind
     ) == [(), ('"a"',), ('"b"',), ('"C"',)]
 
 
+@pytest.mark.modin_sp_short_regress
 @pytest.mark.parametrize(
     "labels, expected_identifiers",
     [
@@ -460,6 +476,7 @@ def test_get_snowflake_quoted_identifiers_by_pandas_labels(
     )
 
 
+@pytest.mark.modin_sp_short_regress
 @pytest.mark.parametrize(
     "labels, expected_identifiers",
     [(["a", "b"], [('"a"', '"d"'), ('"b"',)]), (["C", None], [('"C"',), ('"INDEX"',)])],
@@ -476,6 +493,7 @@ def test_get_snowflake_quoted_identifiers_by_pandas_labels_multiple(
     )
 
 
+@pytest.mark.modin_sp_short_regress
 @pytest.mark.parametrize(
     "labels, expected_identifiers",
     [(["A"], [()]), (["B"], [()]), (["c"], [()])],
@@ -492,6 +510,7 @@ def test_get_snowflake_quoted_identifiers_by_pandas_labels_case_sensitive(
     )
 
 
+@pytest.mark.modin_sp_short_regress
 @pytest.mark.parametrize(
     "labels, expected_identifiers", [(["ABC"], [()]), (["abc"], [()])]
 )
@@ -507,6 +526,7 @@ def test_get_snowflake_quoted_identifiers_by_pandas_labels_missing(
     )
 
 
+@pytest.mark.modin_sp_short_regress
 def test_data_columns_index(test_dataframes, test_dataframes_with_multiindex_on_column):
     assert_index_equal(
         test_dataframes.internal_frame.data_columns_index,
@@ -518,6 +538,7 @@ def test_data_columns_index(test_dataframes, test_dataframes_with_multiindex_on_
     )
 
 
+@pytest.mark.modin_sp_short_regress
 def test_is_multiindex(test_dataframes, test_dataframes_with_multiindex_on_column):
     assert not test_dataframes.internal_frame.is_multiindex(axis=0)
     assert not test_dataframes.internal_frame.is_multiindex(axis=1)
@@ -532,6 +553,7 @@ def test_is_multiindex(test_dataframes, test_dataframes_with_multiindex_on_colum
         test_dataframes.internal_frame.is_multiindex(axis=-1)
 
 
+@pytest.mark.modin_sp_short_regress
 def test_immutability(test_dataframes):
     frame = test_dataframes.internal_frame
 
@@ -565,6 +587,7 @@ def verify_immutability(attr_name: str) -> None:
     assert type(frame.data_column_index_names) == tuple
 
 
+@pytest.mark.modin_sp_short_regress
 @pytest.mark.parametrize("level0, level1", [(1, 1), (2, 1), (1, 2), (2, 3)])
 @patch("snowflake.snowpark.dataframe.DataFrame")
 def test_num_levels(mock_dataframe, level0, level1):
@@ -607,6 +630,7 @@ def test_num_levels(mock_dataframe, level0, level1):
     assert frame.num_index_levels(axis=1) == level1
 
 
+@pytest.mark.modin_sp_short_regress
 @pytest.mark.parametrize(
     "pandas_labels, frame_identifier, expected_message",
     [
@@ -693,6 +717,7 @@ def test_validation_duplicated_data_columns_for_labels(
         )
 
 
+@pytest.mark.modin_sp_short_regress
 def test_update_columns_quoted_identifier_with_expressions_negative(test_dataframes):
     with pytest.raises(ValueError, match="is not in"):
         test_dataframes.internal_frame.update_snowflake_quoted_identifiers_with_expressions(

tox.ini

@@ -196,6 +196,8 @@ markers =
     timeout: tests that need a timeout time
     localtest: local tests
     scala: scala tests
+    modin_sp_short_regress: modin short_regress tests run in sproc
+    modin_sp_precommit: modin precommit tests run in sproc
 addopts = --doctest-modules --timeout=1200
 
 [flake8]