SNOW-909719: enable udf test for python 3.11 (#1079)

.github/workflows/precommit.yml

@@ -101,18 +101,18 @@ jobs:
         run: python -m pip install -U setuptools pip wheel
       - name: Install tox
         run: python -m pip install tox
-      - if: ${{ matrix.python-version != '3.11' && contains('macos', matrix.os.download_name) }}
+      - if: ${{ !(matrix.python-version == '3.11' && matrix.cloud-provider == 'gcp') && contains('macos', matrix.os.download_name) }}
         name: Run doctests
         run: python -m tox -e "py${PYTHON_VERSION}-doctest-notudf-ci"
         env:
           PYTHON_VERSION: ${{ matrix.python-version }}
           cloud_provider: ${{ matrix.cloud-provider }}
           PYTEST_ADDOPTS: --color=yes --tb=short
           TOX_PARALLEL_NO_SPINNER: 1
-          # Specify SNOWFLAKE_IS_PYTHON_RUNTIME_TEST: 1 when adding >= python3.11 with no server-side support
+          # Specify SNOWFLAKE_IS_PYTHON_RUNTIME_TEST: 1 when adding >= python3.12 with no server-side support
           # For example, see https://github.com/snowflakedb/snowpark-python/pull/681
         shell: bash
-      - if: ${{ matrix.python-version != '3.11' }}
+      - if: ${{ !(matrix.python-version == '3.11' && matrix.cloud-provider == 'gcp') }}
         name: Run tests (excluding doctests)
         run: python -m tox -e "py${PYTHON_VERSION/\./}-notdoctest-ci"
         env:
@@ -121,7 +121,7 @@ jobs:
           PYTEST_ADDOPTS: --color=yes --tb=short
           TOX_PARALLEL_NO_SPINNER: 1
         shell: bash
-      - if: ${{ matrix.python-version == '3.11' }}
+      - if: ${{ matrix.python-version == '3.11' && matrix.cloud-provider == 'gcp' }}
         name: Run tests (excluding udf and doctest tests)
         run: python -m tox -e "py${PYTHON_VERSION/\./}-notudfdoctest"
         env:
@@ -222,7 +222,7 @@ jobs:
         run: python -m pip install -U setuptools pip wheel
       - name: Install tox
         run: python -m pip install tox
-      - if: ${{ matrix.python-version != '3.11' && contains('macos', matrix.os.download_name) }}
+      - if: ${{ !(matrix.python-version == '3.11' && matrix.cloud-provider == 'gcp') && contains('macos', matrix.os.download_name) }}
         name: Run doctests
         run: python -m tox -e "py${PYTHON_VERSION}-doctest-notudf-ci"
         env:
@@ -231,7 +231,7 @@ jobs:
           PYTEST_ADDOPTS: --color=yes --tb=short --disable_sql_simplifier
           TOX_PARALLEL_NO_SPINNER: 1
         shell: bash
-      - if: ${{ matrix.python-version != '3.11' }}
+      - if: ${{ !(matrix.python-version == '3.11' && matrix.cloud-provider == 'gcp') }}
         name: Run tests (excluding doctests)
         run: python -m tox -e "py${PYTHON_VERSION/\./}-notdoctest-ci"
         env:
@@ -240,7 +240,7 @@ jobs:
           PYTEST_ADDOPTS: --color=yes --tb=short --disable_sql_simplifier
           TOX_PARALLEL_NO_SPINNER: 1
         shell: bash
-      - if: ${{ matrix.python-version == '3.11' }}
+      - if: ${{ matrix.python-version == '3.11' && matrix.cloud-provider == 'gcp' }}
         name: Run tests (excluding udf and doctest tests)
         run: python -m tox -e "py${PYTHON_VERSION/\./}-notudfdoctest"
         env:
@@ -308,7 +308,7 @@ jobs:
           cloud_provider: ${{ matrix.cloud-provider }}
           PYTEST_ADDOPTS: --color=yes --tb=short
           TOX_PARALLEL_NO_SPINNER: 1
-          # Specify SNOWFLAKE_IS_PYTHON_RUNTIME_TEST: 1 when adding >= python3.11 with no server-side support
+          # Specify SNOWFLAKE_IS_PYTHON_RUNTIME_TEST: 1 when adding >= python3.12 with no server-side support
           # For example, see https://github.com/snowflakedb/snowpark-python/pull/681
         shell: bash
       - name: Combine coverages

CHANGELOG.md

@@ -2,6 +2,10 @@
 
 ## 1.9.0 (TBD)
 
+### New Features
+
+- Added support for the Python 3.11 runtime environment.
+
 ### Dependency updates
 
 - Added back the dependency of `typing-extensions`.

CONTRIBUTING.md

@@ -33,7 +33,7 @@ cd snowpark-python
 
 #### Install the library in edit mode and install its dependencies
 
-- Create a new Python virtual environment with any Python version that we support. Currently supported version is **Python 3.8**. Support for Python 3.9 and 3.10 is available as a preview feature to all accounts. For example,
+- Create a new Python virtual environment with any Python version that we support. Currently supported version is **Python 3.8, Python 3.9, Python 3.10**. Support for Python 3.11 is available as a preview feature to all accounts. For example,
 
   ```bash
   conda create --name snowpark-dev python=3.8

README.md

@@ -18,7 +18,7 @@ If you don't have a Snowflake account yet, you can [sign up for a 30-day free tr
 
 ### Create a Python virtual environment
 You can use [miniconda][miniconda], [anaconda][anaconda], or [virtualenv][virtualenv]
-to create a Python 3.8, 3.9 or 3.10 virtual environment.
+to create a Python 3.8, 3.9, 3.10 or 3.11 virtual environment.
 
 To have the best experience when using it with UDFs, [creating a local conda environment with the Snowflake channel][use snowflake channel] is recommended.
 

setup.py

@@ -19,22 +19,14 @@
     "typing-extensions>=4.1.0, <5.0.0",
     "pyyaml",
 ]
-CLOUDPICKLE_REQ_LIST = ["cloudpickle>=1.6.0,<=2.0.0"]
-REQUIRED_PYTHON_VERSION = ">=3.8, <3.11"
+CLOUDPICKLE_REQ_LIST = [
+    "cloudpickle>=1.6.0,<=2.0.0;python_version<'3.11'",
+    "cloudpickle==2.2.1;python_version~='3.11'",  # backend only supports cloudpickle 2.2.1 + python 3.11 at the moment
+]
+REQUIRED_PYTHON_VERSION = ">=3.8, <3.12"
 
 if os.getenv("SNOWFLAKE_IS_PYTHON_RUNTIME_TEST", False):
-    # this allows us to update the cloudpickle dependency by env var when building package privately
-    # without re-releasing package just to update the dependency, example if we want to lower cloudpickle dep to 2.1.0:
-    # SNOWFLAKE_IS_PYTHON_RUNTIME_TEST=1 CLOUDPICKLE_PYTHON_311_DEPENDENCY="cloudpickle==2.1.0;python_version~='3.11'" pip install .
-    CLOUDPICKLE_PYTHON_311_DEPENDENCY = os.getenv(
-        "CLOUDPICKLE_PYTHON_311_DEPENDENCY",
-        "cloudpickle==2.2.1;python_version~='3.11'",
-    )
     REQUIRED_PYTHON_VERSION = ">=3.8"
-    CLOUDPICKLE_REQ_LIST = [
-        "cloudpickle>=1.6.0,<=2.0.0;python_version<'3.11'",
-        CLOUDPICKLE_PYTHON_311_DEPENDENCY,
-    ]
 
 INSTALL_REQ_LIST.extend(CLOUDPICKLE_REQ_LIST)
 
@@ -114,6 +106,7 @@
         "Programming Language :: Python :: 3.8",
         "Programming Language :: Python :: 3.9",
         "Programming Language :: Python :: 3.10",
+        "Programming Language :: Python :: 3.11",
         "Topic :: Database",
         "Topic :: Software Development",
         "Topic :: Software Development :: Libraries",

src/snowflake/snowpark/session.py

@@ -822,7 +822,7 @@ def add_packages(
             >>> # add numpy with the latest version on Snowflake Anaconda
             >>> # and pandas with the version "1.3.*"
             >>> # and dateutil with the local version in your environment
-            >>> session.add_packages("numpy", "pandas==1.4.*", dateutil)
+            >>> session.add_packages("numpy", "pandas==1.5.*", dateutil)
             >>> @udf
             ... def get_package_name_udf() -> list:
             ...     return [numpy.__name__, pandas.__name__, dateutil.__name__]

tests/integ/test_packaging.py

@@ -762,13 +762,37 @@ def test_add_requirements_with_empty_stage_as_cache_path(
 
     udf_name = Utils.random_name_for_temp_object(TempObjectType.FUNCTION)
 
-    @udf(name=udf_name, packages=["snowflake-snowpark-python==1.3.0"])
-    def get_numpy_pandas_version() -> str:
-        import snowflake.snowpark as snowpark
+    try:
+
+        @udf(name=udf_name, packages=["snowflake-snowpark-python==1.8.0"])
+        def get_numpy_pandas_version() -> str:
+            import snowflake.snowpark as snowpark
+
+            return f"{snowpark.__version__}"
+
+        Utils.check_answer(session.sql(f"select {udf_name}()"), [Row("1.8.0")])
+    except RuntimeError as e:
+        # this is to temporarily bypass the issue that snowflake-snowpark-python==1.8.0 is not available on gcp
+        # for gcp, python == 3.11, we skip the test
+        # for gcp, python < 3.11, we use older snowpark-python lib
+        assert (
+            "Unable to auto-upload packages: ['snowflake-snowpark-python==1.8.0']"
+            in str(e)
+        )
+        system_version = f"{sys.version_info[0]}.{sys.version_info[1]}"
+        if system_version == "3.11":
+            pytest.xfail(
+                "SNOW-938998: package snowflake-snowpark-python 1.8.0 is not available on gcp yet"
+            )
+        else:
+
+            @udf(name=udf_name, packages=["snowflake-snowpark-python==1.3.0"])
+            def get_numpy_pandas_version() -> str:
+                import snowflake.snowpark as snowpark
 
-        return f"{snowpark.__version__}"
+                return f"{snowpark.__version__}"
 
-    Utils.check_answer(session.sql(f"select {udf_name}()"), [Row("1.3.0")])
+            Utils.check_answer(session.sql(f"select {udf_name}()"), [Row("1.3.0")])
 
 
 @pytest.mark.udf