Run E2E tests on windows runners (#1241)

.github/workflows/build.yaml

@@ -16,12 +16,12 @@ concurrency:
 
 jobs:
   build:
-    name: Build project
-    runs-on: ${{ matrix.os }}
+    needs: define-matrix
     strategy:
+      fail-fast: true
       matrix:
-        os: [ macos-latest, ubuntu-latest, windows-latest ]
-        python-version: [ "3.8", "3.10", "3.11", "3.12" ]
+        os: ${{ fromJSON(needs.define-matrix.outputs.os) }}
+        python-version: ${{ fromJSON(needs.define-matrix.outputs.python) }}
     steps:
       - uses: actions/checkout@v4
         with:

.github/workflows/matrix.yaml

@@ -0,0 +1,34 @@
+name: shared matrix
+on:
+  workflow_call:
+    # Map the workflow outputs to job outputs
+    outputs:
+      os:
+        value: ${{ jobs.define-matrix.outputs.os }}
+      python:
+        value: ${{ jobs.define-matrix.outputs.python }}
+
+jobs:
+  define-matrix:
+    runs-on: ubuntu-latest
+    outputs:
+      os:  ${{ steps.os.outputs.os }}
+      python:  ${{ steps.python.outputs.python }}
+    steps:
+      - name: Define OS versions
+        id: os
+        run: |
+          if [ "${{ github.event_name }}" = "schedule" ]; then
+            echo 'os=["windows-latest", "ubuntu-latest", "macos-latest"]' >> "$GITHUB_OUTPUT"
+          else
+            echo 'os=["windows-latest", "ubuntu-latest"]' >> "$GITHUB_OUTPUT"
+          fi
+      - name: Define python versions
+        id: python
+        run: |
+          if [ "${{ github.event_name }}" = "schedule" ]; then
+            echo 'python=["3.8", "3.9", "3.10", "3.11", "3.12"]' >> "$GITHUB_OUTPUT"
+          else
+            # Last supported and most frequently used
+            echo 'python=["3.8", "3.10"]' >> "$GITHUB_OUTPUT"
+          fi

.github/workflows/test_e2e.yaml

@@ -9,22 +9,32 @@ on:
       - "v*" # Push events to matching v*, i.e. v1.0, v20.15.10
     branches:
       - main
+  schedule:
+    - cron: "0 8 * * *"
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
   cancel-in-progress: true
 
 jobs:
+  define-matrix:
+    uses: ./.github/workflows/matrix.yaml
   tests:
-    runs-on: ubuntu-latest
+    needs: define-matrix
+    strategy:
+      fail-fast: true
+      matrix:
+        os: ${{ fromJSON(needs.define-matrix.outputs.os) }}
+        python-version: ${{ fromJSON(needs.define-matrix.outputs.python) }}
+    runs-on: ${{ matrix.os }}
     steps:
       - uses: actions/checkout@v4
         with:
           persist-credentials: false
       - name: Set up Python
         uses: actions/setup-python@v5
         with:
-          python-version: "3.11"
+          python-version: ${{ matrix.python-version }}
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip hatch

.github/workflows/test_integration.yaml

@@ -9,17 +9,24 @@ on:
       - "v*" # Push events to matching v*, i.e. v1.0, v20.15.10
     branches:
       - main
+  schedule:
+    - cron: "0 8 * * *"
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
   cancel-in-progress: true
 
 
 jobs:
+  define-matrix:
+    uses: ./.github/workflows/matrix.yaml
   tests:
+    needs: define-matrix
     strategy:
+      fail-fast: true
       matrix:
-        os: [ ubuntu-latest, windows-latest ]
+        os: ${{ fromJSON(needs.define-matrix.outputs.os) }}
+        python-version: ${{ fromJSON(needs.define-matrix.outputs.python) }}
     runs-on: ${{ matrix.os }}
     steps:
       - uses: actions/checkout@v4

pyproject.toml

@@ -147,7 +147,7 @@ select = [
 ]
 
 [tool.pytest.ini_options]
-addopts = "-m 'not integration and not performance and not e2e and not spcs and not loaded_modules and not integration_experimental'"
+addopts = "-vv --maxfail=10 -m 'not integration and not performance and not e2e and not spcs and not loaded_modules and not integration_experimental'"
 markers = [
   "integration: mark test as integration test",
   "performance: mark test as performance test",

tests_e2e/__snapshots__/test_error_handling.ambr

@@ -1,10 +1,10 @@
 # serializer version: 1
 # name: test_corrupted_config_in_default_location
   '''
-  ╭─ Error ──────────────────────────────────────────────────────────────────────╮
-  │ Configuration file seems to be corrupted. Key "demo" already exists. at line │
-  │ 2 col 18                                                                     │
-  ╰──────────────────────────────────────────────────────────────────────────────╯
+  +- Error ----------------------------------------------------------------------+
+  | Configuration file seems to be corrupted. Key "demo" already exists. at line |
+  | 2 col 18                                                                     |
+  +------------------------------------------------------------------------------+
 
   '''
 # ---

tests_e2e/__snapshots__/test_installation.ambr

@@ -16,34 +16,34 @@
 
    Snowflake CLI tool for developers.                                             
 
-  ╭─ Options ────────────────────────────────────────────────────────────────────╮
-  │ --version                           Shows version of the Snowflake CLI       │
-  │ --info                              Shows information about the Snowflake    │
-  │                                     CLI                                      │
-  │ --config-file                 FILE  Specifies Snowflake CLI configuration    │
-  │                                     file that should be used                 │
-  │                                     [default: None]                          │
-  │ --install-completion                Install completion for the current       │
-  │                                     shell.                                   │
-  │ --show-completion                   Show completion for the current shell,   │
-  │                                     to copy it or customize the              │
-  │                                     installation.                            │
-  │ --help                -h            Show this message and exit.              │
-  ╰──────────────────────────────────────────────────────────────────────────────╯
-  ╭─ Commands ───────────────────────────────────────────────────────────────────╮
-  │ app          Manages a Snowflake Native App                                  │
-  │ connection   Manages connections to Snowflake.                               │
-  │ cortex       Provides access to Snowflake Cortex.                            │
-  │ git          Manages git repositories in Snowflake.                          │
-  │ notebook     Manages notebooks in Snowflake.                                 │
-  │ object       Manages Snowflake objects like warehouses and stages            │
-  │ snowpark     Manages procedures and functions.                               │
-  │ spcs         Manages Snowpark Container Services compute pools, services,    │
-  │              image registries, and image repositories.                       │
-  │ sql          Executes Snowflake query.                                       │
-  │ stage        Manages stages.                                                 │
-  │ streamlit    Manages a Streamlit app in Snowflake.                           │
-  ╰──────────────────────────────────────────────────────────────────────────────╯
+  +- Options --------------------------------------------------------------------+
+  | --version                           Shows version of the Snowflake CLI       |
+  | --info                              Shows information about the Snowflake    |
+  |                                     CLI                                      |
+  | --config-file                 FILE  Specifies Snowflake CLI configuration    |
+  |                                     file that should be used                 |
+  |                                     [default: None]                          |
+  | --install-completion                Install completion for the current       |
+  |                                     shell.                                   |
+  | --show-completion                   Show completion for the current shell,   |
+  |                                     to copy it or customize the              |
+  |                                     installation.                            |
+  | --help                -h            Show this message and exit.              |
+  +------------------------------------------------------------------------------+
+  +- Commands -------------------------------------------------------------------+
+  | app          Manages a Snowflake Native App                                  |
+  | connection   Manages connections to Snowflake.                               |
+  | cortex       Provides access to Snowflake Cortex.                            |
+  | git          Manages git repositories in Snowflake.                          |
+  | notebook     Manages notebooks in Snowflake.                                 |
+  | object       Manages Snowflake objects like warehouses and stages            |
+  | snowpark     Manages procedures and functions.                               |
+  | spcs         Manages Snowpark Container Services compute pools, services,    |
+  |              image registries, and image repositories.                       |
+  | sql          Executes Snowflake query.                                       |
+  | stage        Manages stages.                                                 |
+  | streamlit    Manages a Streamlit app in Snowflake.                           |
+  +------------------------------------------------------------------------------+
 
 
   '''

tests_e2e/conftest.py

@@ -13,8 +13,10 @@
 # limitations under the License.
 
 import os
+import platform
 import shutil
 import subprocess
+import sys
 import tempfile
 from contextlib import contextmanager
 from pathlib import Path
@@ -25,6 +27,50 @@
 
 TEST_DIR = Path(__file__).parent
 
+IS_WINDOWS = platform.system() == "Windows"
+
+
+def _clean_output(text: str):
+    """
+    Replacing util to clean up console output. Typer is using rich.Panel to show the --help content.
+    Unfortunately Typer implementation hardcodes box style for Panel.
+    The typer.rich_utils.STYLE_OPTIONS_TABLE_BOX works only for content within the Panel.
+    """
+    if text is None:
+        return None
+    return (
+        text.replace("│", "|")
+        .replace("─", "-")
+        .replace("╭", "+")
+        .replace("╰", "+")
+        .replace("╯", "+")
+        .replace("╮", "+")
+    )
+
+
+def subprocess_check_output(cmd):
+    try:
+        output = subprocess.check_output(
+            cmd, shell=IS_WINDOWS, stderr=sys.stdout, encoding="utf-8"
+        )
+        return _clean_output(output)
+    except subprocess.CalledProcessError as err:
+        print(err.output)
+        raise
+
+
+def subprocess_run(cmd):
+    p = subprocess.run(
+        cmd,
+        shell=IS_WINDOWS,
+        capture_output=True,
+        text=True,
+        encoding="utf-8",
+    )
+    p.stdout = _clean_output(p.stdout)
+    p.stderr = _clean_output(p.stderr)
+    return p
+
 
 @pytest.fixture(scope="session")
 def test_root_path():
@@ -57,7 +103,10 @@ def snowcli(test_root_path):
         _create_venv(tmp_dir_path)
         _build_snowcli(tmp_dir_path, test_root_path)
         _install_snowcli_with_external_plugin(tmp_dir_path, test_root_path)
-        yield tmp_dir_path / "bin" / "snow"
+        if IS_WINDOWS:
+            yield tmp_dir_path / "Scripts" / "snow.exe"
+        else:
+            yield tmp_dir_path / "bin" / "snow"
 
 
 @pytest.fixture(autouse=True)
@@ -66,20 +115,20 @@ def isolate_default_config_location(monkeypatch, temp_dir):
 
 
 def _create_venv(tmp_dir: Path) -> None:
-    subprocess.check_call(["python", "-m", "venv", tmp_dir])
+    subprocess_check_output(["python", "-m", "venv", tmp_dir])
 
 
 def _build_snowcli(venv_path: Path, test_root_path: Path) -> None:
-    subprocess.check_call(
-        [_python_path(venv_path), "-m", "pip", "install", "--upgrade", "build"]
+    subprocess_check_output(
+        [_python_path(venv_path), "-m", "pip", "install", "--upgrade", "build"],
     )
-    subprocess.check_call(
+    subprocess_check_output(
         [_python_path(venv_path), "-m", "build", test_root_path / ".."]
     )
 
 
 def _pip_install(python, *args):
-    return subprocess.check_call([python, "-m", "pip", "install", *args])
+    return subprocess_check_output([python, "-m", "pip", "install", *args])
 
 
 def _install_snowcli_with_external_plugin(
@@ -103,6 +152,8 @@ def _install_snowcli_with_external_plugin(
 
 
 def _python_path(venv_path: Path) -> Path:
+    if IS_WINDOWS:
+        return venv_path / "Scripts" / "python.exe"
     return venv_path / "bin" / "python"
 
 

tests_e2e/test_error_handling.py

@@ -13,19 +13,20 @@
 # limitations under the License.
 
 import os
-import subprocess
 from pathlib import Path
 
 import pytest
 
+from tests_e2e.conftest import subprocess_run
+
 
 @pytest.mark.e2e
 def test_error_traceback_disabled_without_debug(snowcli, test_root_path):
     config_path = test_root_path / "config" / "config.toml"
     os.chmod(config_path, 0o700)
 
     traceback_msg = "Traceback (most recent call last)"
-    result = subprocess.run(
+    result = subprocess_run(
         [
             snowcli,
             "--config-file",
@@ -35,17 +36,14 @@ def test_error_traceback_disabled_without_debug(snowcli, test_root_path):
             "integration",
             "-q",
             "select foo",
-        ],
-        capture_output=True,
-        text=True,
+        ]
     )
 
-    assert result.returncode == 1
     assert "SQL compilation error" in result.stdout
     assert traceback_msg not in result.stdout
     assert not result.stderr
 
-    result_debug = subprocess.run(
+    result_debug = subprocess_run(
         [
             snowcli,
             "--config-file",
@@ -56,12 +54,9 @@ def test_error_traceback_disabled_without_debug(snowcli, test_root_path):
             "-q",
             "select foo",
             "--debug",
-        ],
-        capture_output=True,
-        text=True,
+        ]
     )
 
-    assert result_debug.returncode == 1
     assert result_debug.stdout == "select foo\n"
     assert traceback_msg in result_debug.stderr
 
@@ -74,20 +69,14 @@ def test_corrupted_config_in_default_location(
     default_config.write_text("[connections.demo]\n[connections.demo]")
     default_config.chmod(0o600)
     # corrupted config should produce human-friendly error
-    result_err = subprocess.run(
+    result_err = subprocess_run(
         [snowcli, "connection", "list"],
-        capture_output=True,
-        text=True,
     )
-    assert result_err.returncode == 1
     assert result_err.stderr == snapshot
 
     # corrupted config in default location should not influence one passed with --config-file flag
     healthy_config = test_root_path / "config" / "config.toml"
-    result_healthy = subprocess.run(
+    result_healthy = subprocess_run(
         [snowcli, "--config-file", healthy_config, "connection", "list"],
-        capture_output=True,
-        text=True,
     )
-    assert result_healthy.returncode == 0, result_healthy.stderr
     assert "dev" in result_healthy.stdout and "integration" in result_healthy.stdout