From 0dc40bb108b85089a05a7800d9eb3e6b446230f8 Mon Sep 17 00:00:00 2001 From: Adam Stus Date: Tue, 30 Jul 2024 12:39:36 +0200 Subject: [PATCH] Changed integration tests user auth to private key --- .github/workflows/cleanup.yaml | 18 +- .github/workflows/test_cli_action.yaml | 62 +++-- .github/workflows/test_e2e.yaml | 252 +++++++++--------- .github/workflows/test_fork.yaml | 108 ++++++++ .github/workflows/test_integration.yaml | 88 +----- .github/workflows/test_trusted.yaml | 81 ++++++ .github/workflows/test_ubuntu_macos.yaml | 48 ++++ .github/workflows/test_windows.yaml | 48 ++++ conftest.py | 1 - scripts/cleanup.py | 5 +- src/snowflake/cli/app/snow_connector.py | 24 +- tests_integration/snowflake_connector.py | 10 +- .../test_temporary_connection.py | 8 +- 13 files changed, 495 insertions(+), 258 deletions(-) create mode 100644 .github/workflows/test_fork.yaml create mode 100644 .github/workflows/test_trusted.yaml create mode 100644 .github/workflows/test_ubuntu_macos.yaml create mode 100644 .github/workflows/test_windows.yaml diff --git a/.github/workflows/cleanup.yaml b/.github/workflows/cleanup.yaml index c39310d367..3164122fd0 100644 --- a/.github/workflows/cleanup.yaml +++ b/.github/workflows/cleanup.yaml @@ -12,19 +12,9 @@ jobs: - uses: actions/checkout@v4 with: persist-credentials: false - - name: Set up Python - uses: actions/setup-python@v5 + - name: Run cleanup + uses: ./.github/workflows/test_integration_ubuntu.yaml with: python-version: "3.11" - - name: Install dependencies - run: | - python -m pip install --upgrade pip hatch - python -m hatch env create e2e - - name: Run cleaunp - env: - TERM: unknown - SNOWFLAKE_CONNECTIONS_INTEGRATION_HOST: ${{ secrets.SNOWFLAKE_HOST }} - SNOWFLAKE_CONNECTIONS_INTEGRATION_USER: ${{ secrets.SNOWFLAKE_USER }} - SNOWFLAKE_CONNECTIONS_INTEGRATION_PASSWORD: ${{ secrets.SNOWFLAKE_PASSWORD }} - SNOWFLAKE_CONNECTIONS_INTEGRATION_ACCOUNT: ${{ secrets.SNOWFLAKE_ACCOUNT }} - run: python -m hatch run e2e:cleanup + python-env: e2e + hatch-run: e2e:cleanup diff --git a/.github/workflows/test_cli_action.yaml b/.github/workflows/test_cli_action.yaml index b40166b4cf..4134d8b8c2 100644 --- a/.github/workflows/test_cli_action.yaml +++ b/.github/workflows/test_cli_action.yaml @@ -1,27 +1,35 @@ -name: "CLI Action testing" - -on: - schedule: - - cron: "0 0 * * *" - workflow_dispatch: - -jobs: - version: - name: "Check Snowflake CLI version" - runs-on: ubuntu-latest - steps: - - name: Checkout repo - uses: actions/checkout@v4 - with: - persist-credentials: false - - uses: Snowflake-Labs/snowflake-cli-action@v1 - with: - cli-version: "latest" - default-config-file-path: "tests_integration/config/connection_configs.toml" - - name: Test connection - env: - TERM: unknown - SNOWFLAKE_CONNECTIONS_INTEGRATION_USER: ${{ secrets.SNOWFLAKE_USER }} - SNOWFLAKE_CONNECTIONS_INTEGRATION_PASSWORD: ${{ secrets.SNOWFLAKE_PASSWORD }} - SNOWFLAKE_CONNECTIONS_INTEGRATION_ACCOUNT: ${{ secrets.SNOWFLAKE_ACCOUNT }} - run: snow connection test -c integration | grep Status +#name: "CLI Action testing" +# +#on: +# schedule: +# - cron: "0 0 * * *" +# workflow_dispatch: +# +#jobs: +# version: +# name: "Check Snowflake CLI version" +# runs-on: ubuntu-latest +# steps: +# - name: Checkout repo +# uses: actions/checkout@v4 +# with: +# persist-credentials: false +# - uses: Snowflake-Labs/snowflake-cli-action@v1 +# with: +# cli-version: "latest" +# default-config-file-path: "tests_integration/config/connection_configs.toml" +# - name: Set workspace parent directory +# run: echo "PARENT_DIR=$(dirname "${{ github.workspace }}")" >> $GITHUB_ENV +# - name: Test connection +# env: +# TERM: unknown +# SNOWFLAKE_CONNECTIONS_INTEGRATION_AUTHENTICATOR: SNOWFLAKE_JWT +# SNOWFLAKE_CONNECTIONS_INTEGRATION_USER: ${{ secrets.SNOWFLAKE_USER }} +# SNOWFLAKE_CONNECTIONS_INTEGRATION_ACCOUNT: ${{ secrets.SNOWFLAKE_ACCOUNT }} +# SNOWFLAKE_CONNECTIONS_INTEGRATION_PRIVATE_KEY_PATH: ${{ env.PARENT_DIR }}/.ssh/key.p8 +# SNOWFLAKE_CONNECTIONS_INTEGRATION_PRIVATE_KEY: ${{ secrets.SNOWFLAKE_PRIVATE_KEY }} +# run: | +# mkdir ${{ env.PARENT_DIR }}/.ssh +# echo "${SNOWFLAKE_PRIVATE_KEY}" > ${{ env.PARENT_DIR }}/.ssh/key.p8 +# sudo chmod 600 ${{ env.PARENT_DIR }}/.ssh/key.p8 +# snow connection test -c integration | grep Status diff --git a/.github/workflows/test_e2e.yaml b/.github/workflows/test_e2e.yaml index 7d881cb323..445e65934f 100644 --- a/.github/workflows/test_e2e.yaml +++ b/.github/workflows/test_e2e.yaml @@ -1,127 +1,125 @@ -name: E2E testing - -on: - pull_request: - branches: - - "*" - push: - tags: - - "v*" # Push events to matching v*, i.e. v1.0, v20.15.10 - branches: - - main - repository_dispatch: - types: [ok-to-test-command] - schedule: - - cron: "0 8 * * *" - -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true - -jobs: - define-matrix: - uses: ./.github/workflows/matrix.yaml - - e2e-trusted: - needs: define-matrix - strategy: - fail-fast: true - matrix: - os: ${{ fromJSON(needs.define-matrix.outputs.os) }} - python-version: ${{ fromJSON(needs.define-matrix.outputs.python) }} - runs-on: ${{ matrix.os }} - if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository - steps: - - uses: actions/checkout@v4 - with: - persist-credentials: false - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: ${{ matrix.python-version }} - - name: Install dependencies - run: | - python -m pip install --upgrade pip hatch - python -m hatch env create e2e - - name: Run end to end tests - env: - TERM: unknown - SNOWFLAKE_CONNECTIONS_INTEGRATION_HOST: ${{ secrets.SNOWFLAKE_HOST }} - SNOWFLAKE_CONNECTIONS_INTEGRATION_USER: ${{ secrets.SNOWFLAKE_USER }} - SNOWFLAKE_CONNECTIONS_INTEGRATION_PASSWORD: ${{ secrets.SNOWFLAKE_PASSWORD }} - SNOWFLAKE_CONNECTIONS_INTEGRATION_ACCOUNT: ${{ secrets.SNOWFLAKE_ACCOUNT }} - run: python -m hatch run e2e:test - - # Repo owner has commented /ok-to-test on a (fork-based) pull request - e2e-fork: - needs: define-matrix - strategy: - fail-fast: true - matrix: - os: ${{ fromJSON(needs.define-matrix.outputs.os) }} - python-version: ${{ fromJSON(needs.define-matrix.outputs.python) }} - runs-on: ${{ matrix.os }} - permissions: - pull-requests: write - checks: write - if: | - github.event_name == 'repository_dispatch' && - github.event.client_payload.slash_command.args.named.sha != '' && - contains( - github.event.client_payload.pull_request.head.sha, - github.event.client_payload.slash_command.args.named.sha - ) - steps: - - uses: actions/checkout@v4 - with: - persist-credentials: false - ref: 'refs/pull/${{ github.event.client_payload.pull_request.number }}/merge' - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: ${{ matrix.python-version }} - - name: Install dependencies - run: | - python -m pip install --upgrade pip hatch - python -m hatch env create e2e - - name: Run end to end tests - env: - TERM: unknown - SNOWFLAKE_CONNECTIONS_INTEGRATION_HOST: ${{ secrets.SNOWFLAKE_HOST }} - SNOWFLAKE_CONNECTIONS_INTEGRATION_USER: ${{ secrets.SNOWFLAKE_USER }} - SNOWFLAKE_CONNECTIONS_INTEGRATION_PASSWORD: ${{ secrets.SNOWFLAKE_PASSWORD }} - SNOWFLAKE_CONNECTIONS_INTEGRATION_ACCOUNT: ${{ secrets.SNOWFLAKE_ACCOUNT }} - run: python -m hatch run e2e:test - # Update check run called "e2e-fork" - - uses: actions/github-script@v7 - id: update-check-run - if: ${{ always() }} - env: - number: ${{ github.event.client_payload.pull_request.number }} - job: ${{ github.job }} - # Conveniently, job.status maps to https://developer.github.com/v3/checks/runs/#update-a-check-run - conclusion: ${{ job.status }} - with: - github-token: ${{ secrets.GITHUB_TOKEN }} - script: | - const { data: pull } = await github.rest.pulls.get({ - ...context.repo, - pull_number: process.env.number - }); - const ref = pull.head.sha; - - const { data: checks } = await github.rest.checks.listForRef({ - ...context.repo, - ref - }); - - const check = checks.check_runs.filter(c => c.name === process.env.job); - - const { data: result } = await github.rest.checks.update({ - ...context.repo, - check_run_id: check[0].id, - status: 'completed', - conclusion: process.env.conclusion - }); - - return result; +#name: E2E testing +# +#on: +# pull_request: +# branches: +# - "*" +# push: +# tags: +# - "v*" # Push events to matching v*, i.e. v1.0, v20.15.10 +# branches: +# - main +# repository_dispatch: +# types: [ok-to-test-command] +# schedule: +# - cron: "0 8 * * *" +# +#concurrency: +# group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} +# cancel-in-progress: true +# +#jobs: +# define-matrix: +# uses: ./.github/workflows/matrix.yaml +# +# e2e-trusted: +# needs: define-matrix +# strategy: +# fail-fast: true +# matrix: +# os: ${{ fromJSON(needs.define-matrix.outputs.os) }} +# python-version: ${{ fromJSON(needs.define-matrix.outputs.python) }} +# runs-on: ubuntu-latest +# if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository +# steps: +# - uses: actions/checkout@v4 +# with: +# persist-credentials: false +# - name: E2E Ubuntu / MacOS +# if: matrix.os == 'ubuntu-latest' || matrix.os == 'macos-latest' +# uses: ./.github/workflows/test_ubuntu_macos.yaml +# with: +# runs-on: ${{ matrix.os }} +# python-version: ${{ matrix.python-version }} +# python-env: e2e +# hatch-run: e2e:test +# - name: E2E Windows +# if: matrix.os == 'windows-latest' +# uses: ./.github/workflows/test_windows.yaml +# with: +# python-version: ${{ matrix.python-version }} +# python-env: e2e +# hatch-run: e2e:test +# +# +# # Repo owner has commented /ok-to-test on a (fork-based) pull request +# e2e-fork: +# needs: define-matrix +# strategy: +# fail-fast: true +# matrix: +# os: ${{ fromJSON(needs.define-matrix.outputs.os) }} +# python-version: ${{ fromJSON(needs.define-matrix.outputs.python) }} +# runs-on: ${{ matrix.os }} +# permissions: +# pull-requests: write +# checks: write +# if: | +# github.event_name == 'repository_dispatch' && +# github.event.client_payload.slash_command.args.named.sha != '' && +# contains( +# github.event.client_payload.pull_request.head.sha, +# github.event.client_payload.slash_command.args.named.sha +# ) +# steps: +# - uses: actions/checkout@v4 +# with: +# persist-credentials: false +# - name: E2E Ubuntu / MacOS +# if: matrix.os == 'ubuntu-latest' || matrix.os == 'macos-latest' +# uses: ./.github/workflows/test_ubuntu_macos.yaml +# with: +# runs-on: ${{ matrix.os }} +# python-version: ${{ matrix.python-version }} +# python-env: e2e +# hatch-run: e2e:test +# - name: E2E Windows +# if: matrix.os == 'windows-latest' +# uses: ./.github/workflows/test_windows.yaml +# with: +# python-version: ${{ matrix.python-version }} +# python-env: e2e +# hatch-run: e2e:test +# # Update check run called "e2e-fork" +# - uses: actions/github-script@v7 +# id: update-check-run +# if: ${{ always() }} +# env: +# number: ${{ github.event.client_payload.pull_request.number }} +# job: ${{ github.job }} +# # Conveniently, job.status maps to https://developer.github.com/v3/checks/runs/#update-a-check-run +# conclusion: ${{ job.status }} +# with: +# github-token: ${{ secrets.GITHUB_TOKEN }} +# script: | +# const { data: pull } = await github.rest.pulls.get({ +# ...context.repo, +# pull_number: process.env.number +# }); +# const ref = pull.head.sha; +# +# const { data: checks } = await github.rest.checks.listForRef({ +# ...context.repo, +# ref +# }); +# +# const check = checks.check_runs.filter(c => c.name === process.env.job); +# +# const { data: result } = await github.rest.checks.update({ +# ...context.repo, +# check_run_id: check[0].id, +# status: 'completed', +# conclusion: process.env.conclusion +# }); +# +# return result; diff --git a/.github/workflows/test_fork.yaml b/.github/workflows/test_fork.yaml new file mode 100644 index 0000000000..f0b02b18da --- /dev/null +++ b/.github/workflows/test_fork.yaml @@ -0,0 +1,108 @@ +name: Test trusted + +on: + workflow_call: + inputs: + runs-on: + required: true + type: string + python-version: + required: true + type: string + python-env: + required: true + type: string + hatch-run: + required: true + type: string + +jobs: + ubuntu-macos-steps: + runs-on: ${{ inputs.runs-on }} + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: ${{ inputs.python-version }} + - name: Install dependencies + run: | + python -m pip install --upgrade pip hatch + python -m hatch env create ${{ inputs.python-env }} + + # Ubuntu and MacOS specific steps + - name: Set workspace parent directory + if: inputs.runs-on == 'ubuntu-latest' || inputs.runs-on == 'macos-latest' + run: echo "PARENT_DIR=$(dirname "${{ github.workspace }}")" >> $GITHUB_ENV + - name: Run integration tests + if: inputs.runs-on == 'ubuntu-latest' || inputs.runs-on == 'macos-latest' + env: + TERM: unknown + SNOWFLAKE_CONNECTIONS_INTEGRATION_AUTHENTICATOR: SNOWFLAKE_JWT + SNOWFLAKE_CONNECTIONS_INTEGRATION_USER: ${{ secrets.SNOWFLAKE_USER }} + SNOWFLAKE_CONNECTIONS_INTEGRATION_ACCOUNT: ${{ secrets.SNOWFLAKE_ACCOUNT }} + SNOWFLAKE_CONNECTIONS_INTEGRATION_PRIVATE_KEY_PATH: ${{ env.PARENT_DIR }}/.ssh/key.p8 + SNOWFLAKE_CONNECTIONS_INTEGRATION_PRIVATE_KEY: ${{ secrets.SNOWFLAKE_PRIVATE_KEY }} + run: | + mkdir ${{ env.PARENT_DIR }}/.ssh + echo "${SNOWFLAKE_PRIVATE_KEY}" > ${{ env.PARENT_DIR }}/.ssh/key.p8 + sudo chmod 600 ${{ env.PARENT_DIR }}/.ssh/key.p8 + python -m hatch run ${{ inputs.hatch-run }} + + # Windows specific steps + - name: Set workspace parent directory + if: inputs.runs-on == 'windows-latest' + run: | + $parentDir = Split-Path -Parent "${{ github.workspace }}" + echo "PARENT_DIR=$parentDir" >> $env:GITHUB_ENV + shell: pwsh + - name: Run integration tests + if: inputs.runs-on == 'windows-latest' + env: + TERM: unknown + SNOWFLAKE_CONNECTIONS_INTEGRATION_AUTHENTICATOR: SNOWFLAKE_JWT + SNOWFLAKE_CONNECTIONS_INTEGRATION_USER: ${{ secrets.SNOWFLAKE_USER }} + SNOWFLAKE_CONNECTIONS_INTEGRATION_ACCOUNT: ${{ secrets.SNOWFLAKE_ACCOUNT }} + SNOWFLAKE_CONNECTIONS_INTEGRATION_PRIVATE_KEY_PATH: ${{ env.PARENT_DIR }}\\ssh\\key.p8 + SNOWFLAKE_PRIVATE_KEY: ${{ secrets.SNOWFLAKE_PRIVATE_KEY }} + run: | + mkdir ${{ env.PARENT_DIR }}\\ssh + Set-Content -Path "${{ env.PARENT_DIR }}\\ssh\\key.p8" -Value $env:SNOWFLAKE_PRIVATE_KEY + python -m hatch run ${{ inputs.hatch-run }} + shell: pwsh + + # Update check run called "integration-fork" + - uses: actions/github-script@v7 + id: update-check-run + if: ${{ always() }} + env: + number: ${{ github.event.client_payload.pull_request.number }} + job: ${{ github.job }} + # Conveniently, job.status maps to https://developer.github.com/v3/checks/runs/#update-a-check-run + conclusion: ${{ job.status }} + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + const { data: pull } = await github.rest.pulls.get({ + ...context.repo, + pull_number: process.env.number + }); + const ref = pull.head.sha; + + const { data: checks } = await github.rest.checks.listForRef({ + ...context.repo, + ref + }); + + const check = checks.check_runs.filter(c => c.name === process.env.job); + + const { data: result } = await github.rest.checks.update({ + ...context.repo, + check_run_id: check[0].id, + status: 'completed', + conclusion: process.env.conclusion + }); + + return result; diff --git a/.github/workflows/test_integration.yaml b/.github/workflows/test_integration.yaml index 9bca9c9164..b3f5fe8ded 100644 --- a/.github/workflows/test_integration.yaml +++ b/.github/workflows/test_integration.yaml @@ -31,28 +31,14 @@ jobs: matrix: os: ${{ fromJSON(needs.define-matrix.outputs.os) }} python-version: ${{ fromJSON(needs.define-matrix.outputs.python) }} - runs-on: ${{ matrix.os }} if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository - steps: - - uses: actions/checkout@v4 - with: - persist-credentials: false - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: ${{ matrix.python-version }} - - name: Install dependencies - run: | - python -m pip install --upgrade pip hatch - python -m hatch env create integration - - name: Run integration tests - env: - TERM: unknown - SNOWFLAKE_CONNECTIONS_INTEGRATION_USER: ${{ secrets.SNOWFLAKE_USER }} - SNOWFLAKE_CONNECTIONS_INTEGRATION_PASSWORD: ${{ secrets.SNOWFLAKE_PASSWORD }} - SNOWFLAKE_CONNECTIONS_INTEGRATION_ACCOUNT: ${{ secrets.SNOWFLAKE_ACCOUNT }} - run: python -m hatch run integration:test - + uses: ./.github/workflows/test_trusted.yaml + with: + runs-on: ${{ matrix.os }} + python-version: ${{ matrix.python-version }} + python-env: integration + hatch-run: integration:test + secrets: inherit # Repo owner has commented /ok-to-test on a (fork-based) pull request integration-fork: @@ -62,7 +48,6 @@ jobs: matrix: os: ${{ fromJSON(needs.define-matrix.outputs.os) }} python-version: ${{ fromJSON(needs.define-matrix.outputs.python) }} - runs-on: ${{ matrix.os }} permissions: pull-requests: write checks: write @@ -73,56 +58,9 @@ jobs: github.event.client_payload.pull_request.head.sha, github.event.client_payload.slash_command.args.named.sha ) - steps: - - uses: actions/checkout@v4 - with: - persist-credentials: false - ref: 'refs/pull/${{ github.event.client_payload.pull_request.number }}/merge' - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: ${{ matrix.python-version }} - - name: Install dependencies - run: | - python -m pip install --upgrade pip hatch - python -m hatch env create integration - - name: Run integration tests - env: - TERM: unknown - SNOWFLAKE_CONNECTIONS_INTEGRATION_USER: ${{ secrets.SNOWFLAKE_USER }} - SNOWFLAKE_CONNECTIONS_INTEGRATION_PASSWORD: ${{ secrets.SNOWFLAKE_PASSWORD }} - SNOWFLAKE_CONNECTIONS_INTEGRATION_ACCOUNT: ${{ secrets.SNOWFLAKE_ACCOUNT }} - run: python -m hatch run integration:test - # Update check run called "integration-fork" - - uses: actions/github-script@v7 - id: update-check-run - if: ${{ always() }} - env: - number: ${{ github.event.client_payload.pull_request.number }} - job: ${{ github.job }} - # Conveniently, job.status maps to https://developer.github.com/v3/checks/runs/#update-a-check-run - conclusion: ${{ job.status }} - with: - github-token: ${{ secrets.GITHUB_TOKEN }} - script: | - const { data: pull } = await github.rest.pulls.get({ - ...context.repo, - pull_number: process.env.number - }); - const ref = pull.head.sha; - - const { data: checks } = await github.rest.checks.listForRef({ - ...context.repo, - ref - }); - - const check = checks.check_runs.filter(c => c.name === process.env.job); - - const { data: result } = await github.rest.checks.update({ - ...context.repo, - check_run_id: check[0].id, - status: 'completed', - conclusion: process.env.conclusion - }); - - return result; + uses: ./.github/workflows/test_fork.yaml + with: + runs-on: ${{ matrix.os }} + python-version: ${{ matrix.python-version }} + python-env: integration + hatch-run: integration:test diff --git a/.github/workflows/test_trusted.yaml b/.github/workflows/test_trusted.yaml new file mode 100644 index 0000000000..d26e9bc54d --- /dev/null +++ b/.github/workflows/test_trusted.yaml @@ -0,0 +1,81 @@ +name: Test trusted + +on: + workflow_call: + inputs: + runs-on: + required: true + type: string + python-version: + required: true + type: string + python-env: + required: true + type: string + hatch-run: + required: true + type: string + +jobs: + ubuntu-macos-steps: + runs-on: ${{ inputs.runs-on }} + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: ${{ inputs.python-version }} + - name: Install dependencies + run: | + python -m pip install --upgrade pip hatch + python -m hatch env create ${{ inputs.python-env }} + + - name: Debug + run: | + echo ${{ inputs.runs-on }} + + # Ubuntu and MacOS specific steps + - name: Set workspace parent directory + if: inputs.runs-on == 'ubuntu-latest' || inputs.runs-on == 'macos-latest' + run: echo "PARENT_DIR=$(dirname "${{ github.workspace }}")" >> $GITHUB_ENV + - name: Run integration tests + if: inputs.runs-on == 'ubuntu-latest' || inputs.runs-on == 'macos-latest' + env: + TERM: unknown + SNOWFLAKE_CONNECTIONS_INTEGRATION_AUTHENTICATOR: SNOWFLAKE_JWT + SNOWFLAKE_CONNECTIONS_INTEGRATION_USER: ${{ secrets.SNOWFLAKE_USER }} + SNOWFLAKE_CONNECTIONS_INTEGRATION_ACCOUNT: ${{ secrets.SNOWFLAKE_ACCOUNT }} + SNOWFLAKE_CONNECTIONS_INTEGRATION_PRIVATE_KEY_PATH: ${{ env.PARENT_DIR }}/.ssh/key.p8 + SNOWFLAKE_CONNECTIONS_INTEGRATION_PRIVATE_KEY: ${{ secrets.SNOWFLAKE_PRIVATE_KEY }} + run: | + mkdir ${{ env.PARENT_DIR }}/.ssh + echo "${SNOWFLAKE_PRIVATE_KEY}" > ${{ env.PARENT_DIR }}/.ssh/key.p8 + sudo chmod 600 ${{ env.PARENT_DIR }}/.ssh/key.p8 + python -m hatch run ${{ inputs.hatch-run }} + + - name: Debug 2 + run: | + echo ${{ inputs.runs-on }} + # Windows specific steps + - name: Set workspace parent directory + if: inputs.runs-on == 'windows-latest' + run: | + $parentDir = Split-Path -Parent "${{ github.workspace }}" + echo "PARENT_DIR=$parentDir" >> $env:GITHUB_ENV + shell: pwsh + - name: Run integration tests + if: inputs.runs-on == 'windows-latest' + env: + TERM: unknown + SNOWFLAKE_CONNECTIONS_INTEGRATION_AUTHENTICATOR: SNOWFLAKE_JWT + SNOWFLAKE_CONNECTIONS_INTEGRATION_USER: ${{ secrets.SNOWFLAKE_USER }} + SNOWFLAKE_CONNECTIONS_INTEGRATION_ACCOUNT: ${{ secrets.SNOWFLAKE_ACCOUNT }} + SNOWFLAKE_CONNECTIONS_INTEGRATION_PRIVATE_KEY_PATH: ${{ env.PARENT_DIR }}\\ssh\\key.p8 + SNOWFLAKE_PRIVATE_KEY: ${{ secrets.SNOWFLAKE_PRIVATE_KEY }} + run: | + mkdir ${{ env.PARENT_DIR }}\\ssh + Set-Content -Path "${{ env.PARENT_DIR }}\\ssh\\key.p8" -Value $env:SNOWFLAKE_PRIVATE_KEY + python -m hatch run ${{ inputs.hatch-run }} + shell: pwsh diff --git a/.github/workflows/test_ubuntu_macos.yaml b/.github/workflows/test_ubuntu_macos.yaml new file mode 100644 index 0000000000..c54ffcb839 --- /dev/null +++ b/.github/workflows/test_ubuntu_macos.yaml @@ -0,0 +1,48 @@ +name: Test Ubuntu / MacOS + +on: + workflow_call: + inputs: + runs-on: + required: true + type: string + python-version: + required: true + type: string + python-env: + required: true + type: string + hatch-run: + required: true + type: string + +jobs: + ubuntu-macos-steps: + runs-on: ${{ inputs.runs-on }} + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: ${{ inputs.python-version }} + - name: Install dependencies + run: | + python -m pip install --upgrade pip hatch + python -m hatch env create ${{ inputs.python-env }} + - name: Set workspace parent directory + run: echo "PARENT_DIR=$(dirname "${{ github.workspace }}")" >> $GITHUB_ENV + - name: Run integration tests + env: + TERM: unknown + SNOWFLAKE_CONNECTIONS_INTEGRATION_AUTHENTICATOR: SNOWFLAKE_JWT + SNOWFLAKE_CONNECTIONS_INTEGRATION_USER: ${{ secrets.SNOWFLAKE_USER }} + SNOWFLAKE_CONNECTIONS_INTEGRATION_ACCOUNT: ${{ secrets.SNOWFLAKE_ACCOUNT }} + SNOWFLAKE_CONNECTIONS_INTEGRATION_PRIVATE_KEY_PATH: ${{ env.PARENT_DIR }}/.ssh/key.p8 + SNOWFLAKE_CONNECTIONS_INTEGRATION_PRIVATE_KEY: ${{ secrets.SNOWFLAKE_PRIVATE_KEY }} + run: | + mkdir ${{ env.PARENT_DIR }}/.ssh + echo "${SNOWFLAKE_PRIVATE_KEY}" > ${{ env.PARENT_DIR }}/.ssh/key.p8 + sudo chmod 600 ${{ env.PARENT_DIR }}/.ssh/key.p8 + python -m hatch run ${{ inputs.hatch-run }} diff --git a/.github/workflows/test_windows.yaml b/.github/workflows/test_windows.yaml new file mode 100644 index 0000000000..ea5e6fd219 --- /dev/null +++ b/.github/workflows/test_windows.yaml @@ -0,0 +1,48 @@ +name: Test Windows + +on: + workflow_call: + inputs: + python-version: + required: true + type: string + python-env: + required: true + type: string + hatch-run: + required: true + type: string + +jobs: + windows-steps: + runs-on: windows-latest + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: ${{ inputs.python-version }} + - name: Install dependencies + run: | + python -m pip install --upgrade pip hatch + python -m hatch env create ${{ inputs.python-env }} + - name: Set workspace parent directory + run: | + $parentDir = Split-Path -Parent "${{ github.workspace }}" + echo "PARENT_DIR=$parentDir" >> $env:GITHUB_ENV + shell: pwsh + - name: Run integration tests + env: + TERM: unknown + SNOWFLAKE_CONNECTIONS_INTEGRATION_AUTHENTICATOR: SNOWFLAKE_JWT + SNOWFLAKE_CONNECTIONS_INTEGRATION_USER: ${{ secrets.SNOWFLAKE_USER }} + SNOWFLAKE_CONNECTIONS_INTEGRATION_ACCOUNT: ${{ secrets.SNOWFLAKE_ACCOUNT }} + SNOWFLAKE_CONNECTIONS_INTEGRATION_PRIVATE_KEY_PATH: ${{ env.PARENT_DIR }}\\ssh\\key.p8 + SNOWFLAKE_PRIVATE_KEY: ${{ secrets.SNOWFLAKE_PRIVATE_KEY }} + run: | + mkdir ${{ env.PARENT_DIR }}\\ssh + Set-Content -Path "${{ env.PARENT_DIR }}\\ssh\\key.p8" -Value $env:SNOWFLAKE_PRIVATE_KEY + python -m hatch run ${{ inputs.hatch-run }} + shell: pwsh diff --git a/conftest.py b/conftest.py index 95aaaa75e9..b72db862c8 100644 --- a/conftest.py +++ b/conftest.py @@ -11,7 +11,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - from unittest import mock import pytest diff --git a/scripts/cleanup.py b/scripts/cleanup.py index 72faceeb1a..d571e08be6 100644 --- a/scripts/cleanup.py +++ b/scripts/cleanup.py @@ -52,9 +52,12 @@ def remove_resources(single: str, plural: str, known_instances: t.List[str], rol role = "INTEGRATION_TESTS" session = Session.builder.configs( { + "authenticator": "SNOWFLAKE_JWT", "account": os.getenv("SNOWFLAKE_CONNECTIONS_INTEGRATION_ACCOUNT"), "user": os.getenv("SNOWFLAKE_CONNECTIONS_INTEGRATION_USER"), - "password": os.getenv("SNOWFLAKE_CONNECTIONS_INTEGRATION_PASSWORD"), + "private_key_path": os.getenv( + "SNOWFLAKE_CONNECTIONS_INTEGRATION_PRIVATE_KEY_PATH" + ), "database": "SNOWCLI_DB", "role": role, } diff --git a/src/snowflake/cli/app/snow_connector.py b/src/snowflake/cli/app/snow_connector.py index 3fe072baa7..ddda5cbb8c 100644 --- a/src/snowflake/cli/app/snow_connector.py +++ b/src/snowflake/cli/app/snow_connector.py @@ -97,7 +97,7 @@ def connect_to_snowflake( k: v for k, v in connection_parameters.items() if v is not None } - connection_parameters = _update_connection_details_with_private_key( + connection_parameters = update_connection_details_with_private_key( connection_parameters ) @@ -163,7 +163,7 @@ def _raise_errors_related_to_session_token( ) -def _update_connection_details_with_private_key(connection_parameters: Dict): +def update_connection_details_with_private_key(connection_parameters: Dict): if "private_key_path" in connection_parameters: if connection_parameters.get("authenticator") == "SNOWFLAKE_JWT": private_key = _load_pem_to_der(connection_parameters["private_key_path"]) @@ -189,13 +189,6 @@ def _load_pem_to_der(private_key_path: str) -> bytes: Given a private key file path (in PEM format), decode key data into DER format """ - from cryptography.hazmat.backends import default_backend - from cryptography.hazmat.primitives.serialization import ( - Encoding, - NoEncryption, - PrivateFormat, - load_pem_private_key, - ) with SecurePath(private_key_path).open( "rb", read_file_limit_mb=DEFAULT_SIZE_LIMIT_MB @@ -222,6 +215,18 @@ def _load_pem_to_der(private_key_path: str) -> bytes: if private_key_pem.startswith(UNENCRYPTED_PKCS8_PK_HEADER): private_key_passphrase = None + return prepare_private_key(private_key_pem, private_key_passphrase) + + +def prepare_private_key(private_key_pem, private_key_passphrase=None): + from cryptography.hazmat.backends import default_backend + from cryptography.hazmat.primitives.serialization import ( + Encoding, + NoEncryption, + PrivateFormat, + load_pem_private_key, + ) + private_key = load_pem_private_key( private_key_pem, ( @@ -231,7 +236,6 @@ def _load_pem_to_der(private_key_path: str) -> bytes: ), default_backend(), ) - return private_key.private_bytes( encoding=Encoding.DER, format=PrivateFormat.PKCS8, diff --git a/tests_integration/snowflake_connector.py b/tests_integration/snowflake_connector.py index feafdb119b..0020083721 100644 --- a/tests_integration/snowflake_connector.py +++ b/tests_integration/snowflake_connector.py @@ -22,6 +22,7 @@ import pytest from snowflake import connector from snowflake.cli.api.exceptions import EnvironmentVariableNotFoundError +from snowflake.cli.app.snow_connector import update_connection_details_with_private_key _ENV_PARAMETER_PREFIX = "SNOWFLAKE_CONNECTIONS_INTEGRATION" SCHEMA_ENV_PARAMETER = f"{_ENV_PARAMETER_PREFIX}_SCHEMA" @@ -89,14 +90,21 @@ def test_role(snowflake_session): def snowflake_session(): config = { "application": "INTEGRATION_TEST", + "authenticator": "SNOWFLAKE_JWT", "account": _get_from_env("ACCOUNT"), "user": _get_from_env("USER"), - "password": _get_from_env("PASSWORD"), + "password": _get_from_env("PASSWORD", allow_none=True), + "private_key_path": _get_from_env("PRIVATE_KEY_PATH", allow_none=True), "host": _get_from_env("HOST", allow_none=True), "warehouse": _get_from_env("WAREHOUSE", allow_none=True), "role": _get_from_env("ROLE", allow_none=True), } + if config["password"] is None and config["private_key_path"] is None: + raise RuntimeError( + "Environment variable SNOWFLAKE_CONNECTIONS_INTEGRATION_PASSWORD or SNOWFLAKE_CONNECTIONS_INTEGRATION_PRIVATE_KEY_PATH is needed." + ) config = {k: v for k, v in config.items() if v is not None} + update_connection_details_with_private_key(config) connection = connector.connect(**config) yield connection connection.close() diff --git a/tests_integration/test_temporary_connection.py b/tests_integration/test_temporary_connection.py index 0e6ac31581..a928b0cd51 100644 --- a/tests_integration/test_temporary_connection.py +++ b/tests_integration/test_temporary_connection.py @@ -27,8 +27,8 @@ "SNOWFLAKE_CONNECTIONS_INTEGRATION_USER": os.environ.get( "SNOWFLAKE_CONNECTIONS_INTEGRATION_USER", None ), - "SNOWFLAKE_PASSWORD": os.environ.get( - "SNOWFLAKE_CONNECTIONS_INTEGRATION_PASSWORD" + "SNOWFLAKE_CONNECTIONS_INTEGRATION_PRIVATE_KEY_PATH": os.environ.get( + "SNOWFLAKE_CONNECTIONS_INTEGRATION_PRIVATE_KEY_PATH", None ), }, clear=True, @@ -41,10 +41,14 @@ def test_temporary_connection(runner, snapshot): "-q", "select 1", "--temporary-connection", + "--authenticator", + "SNOWFLAKE_JWT", "--account", os.environ["SNOWFLAKE_CONNECTIONS_INTEGRATION_ACCOUNT"], "--user", os.environ["SNOWFLAKE_CONNECTIONS_INTEGRATION_USER"], + "--private-key-path", + os.environ["SNOWFLAKE_CONNECTIONS_INTEGRATION_PRIVATE_KEY_PATH"], ] ) assert result.exit_code == 0