From 0c7e9e4f55dce563b407aea18b8ddca529ab4325 Mon Sep 17 00:00:00 2001 From: Afroz Alam Date: Thu, 29 Aug 2024 10:46:53 -0700 Subject: [PATCH 1/3] make use_logical_type option more explicit --- CHANGELOG.md | 2 +- src/snowflake/snowpark/session.py | 16 +++++++++++----- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 42efb25dcb5..6f701cb3e02 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,7 +12,7 @@ #### Improvements -- Added support for `ln` in `snowflake.snowpark.functions` +- Improved documentation for `Session.write_pandas` by making `use_logical_type` option more explicit. - Added support for specifying the following to `DataFrameWriter.save_as_table`: - `enable_schema_evolution` - `data_retention_time` diff --git a/src/snowflake/snowpark/session.py b/src/snowflake/snowpark/session.py index d8a2997fa33..fa21f929a9b 100644 --- a/src/snowflake/snowpark/session.py +++ b/src/snowflake/snowpark/session.py @@ -2393,6 +2393,7 @@ def write_pandas( create_temp_table: bool = False, overwrite: bool = False, table_type: Literal["", "temp", "temporary", "transient"] = "", + use_logical_type: Optional[bool] = None, **kwargs: Dict[str, Any], ) -> Table: """Writes a pandas DataFrame to a table in Snowflake and returns a @@ -2429,6 +2430,11 @@ def write_pandas( table_type: The table type of table to be created. The supported values are: ``temp``, ``temporary``, and ``transient``. An empty string means to create a permanent table. Learn more about table types `here `_. + use_logical_type: Boolean that specifies whether to use Parquet logical types when reading the parquet files + for the unloaded pandas dataframe. With this file format option, Snowflake can interpret Parquet logical + types during data loading. To enable Parquet logical types, set use_logical_type as True. Set to None to + use Snowflakes default. For more information, see: + `file format options: `_. Example:: @@ -2506,11 +2512,11 @@ def write_pandas( + (table_name) ) signature = inspect.signature(write_pandas) - if not ("use_logical_type" in signature.parameters): - # do not pass use_logical_type if write_pandas does not support it - use_logical_type_passed = kwargs.pop("use_logical_type", None) - - if use_logical_type_passed is not None: + use_logical_type_supported = "use_logical_type" in signature.parameters + if use_logical_type_supported: + kwargs["use_logical_type"] = use_logical_type + else: + if use_logical_type is not None: # raise warning to upgrade python connector warnings.warn( "use_logical_type will be ignored because current python " From b4821ea38c9c9bad91a1b1d0191987b71fae19c0 Mon Sep 17 00:00:00 2001 From: Afroz Alam Date: Thu, 29 Aug 2024 13:45:25 -0700 Subject: [PATCH 2/3] fix modin --- src/snowflake/snowpark/session.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/src/snowflake/snowpark/session.py b/src/snowflake/snowpark/session.py index fa21f929a9b..f4fd7eca610 100644 --- a/src/snowflake/snowpark/session.py +++ b/src/snowflake/snowpark/session.py @@ -2511,12 +2511,13 @@ def write_pandas( + (schema + "." if schema else "") + (table_name) ) - signature = inspect.signature(write_pandas) - use_logical_type_supported = "use_logical_type" in signature.parameters - if use_logical_type_supported: - kwargs["use_logical_type"] = use_logical_type - else: - if use_logical_type is not None: + + if use_logical_type is not None: + signature = inspect.signature(write_pandas) + use_logical_type_supported = "use_logical_type" in signature.parameters + if use_logical_type_supported: + kwargs["use_logical_type"] = use_logical_type + else: # raise warning to upgrade python connector warnings.warn( "use_logical_type will be ignored because current python " From 0d318c270c4b2d5860ed401e160859be9097f16c Mon Sep 17 00:00:00 2001 From: Afroz Alam Date: Fri, 30 Aug 2024 15:17:05 -0700 Subject: [PATCH 3/3] Update src/snowflake/snowpark/session.py Co-authored-by: Jamison Rose --- src/snowflake/snowpark/session.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/snowflake/snowpark/session.py b/src/snowflake/snowpark/session.py index f4fd7eca610..c6f430cc980 100644 --- a/src/snowflake/snowpark/session.py +++ b/src/snowflake/snowpark/session.py @@ -2431,7 +2431,7 @@ def write_pandas( and ``transient``. An empty string means to create a permanent table. Learn more about table types `here `_. use_logical_type: Boolean that specifies whether to use Parquet logical types when reading the parquet files - for the unloaded pandas dataframe. With this file format option, Snowflake can interpret Parquet logical + for the uploaded pandas dataframe. With this file format option, Snowflake can interpret Parquet logical types during data loading. To enable Parquet logical types, set use_logical_type as True. Set to None to use Snowflakes default. For more information, see: `file format options: `_.