Sf rebase#1

.github/workflows/build-test.yml

@@ -1,113 +1,46 @@
 name: Build and Test
 
 on:
-    push:
-        braches:
-            - master
-        tags:
-            - v*
-    pull_request:
-        branches:
-            - master
-    schedule:
-      - cron:  '7 3 * * *'
-    workflow_dispatch:
-        inputs:
-          logLevel:
-            default: warning
-            description: "Log level"
-            required: true
-          tags:
-            description: "Test scenario tags"
-
-concurrency:
-  # older builds for the same pull request numer or branch should be cancelled
-  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
-  cancel-in-progress: true
-
+  push:
+    braches:
+      - master
+    tags:
+      - v*
+  pull_request:
+    branches:
+      - master
+  schedule:
+    - cron: "7 3 * * *"
+  workflow_dispatch:
+    inputs:
+      logLevel:
+        default: warning
+        description: "Log level"
+        required: true
+      tags:
+        description: "Test scenario tags"
 jobs:
-    build-test-linux:
-        runs-on: ubuntu-latest
-        strategy:
-            fail-fast: false
-            matrix:
-                cloud: [ 'AWS', 'AZURE', 'GCP' ]
-                go: [ '1.20', '1.19' ]
-        name: ${{ matrix.cloud }} Go ${{ matrix.go }} on Ubuntu
-        steps:
-            - uses: actions/checkout@v1
-            - name: Setup go
-              uses: actions/setup-go@v2
-              with:
-                  go-version: ${{ matrix.go }}
-            - name: Format, Lint
-              shell: bash
-              run: ./ci/build.sh
-            - name: Test
-              shell: bash
-              env:
-                PARAMETERS_SECRET: ${{ secrets.PARAMETERS_SECRET }}
-                CLOUD_PROVIDER: ${{ matrix.cloud }}
-              run: ./ci/test.sh
-            - name: Upload coverage to Codecov
-              uses: codecov/codecov-action@v3
-              with:
-                token: ${{ secrets.CODE_COV_UPLOAD_TOKEN }}
-    build-test-mac:
-        runs-on: macos-latest
-        strategy:
-            fail-fast: false
-            matrix:
-                cloud: [ 'AWS', 'AZURE', 'GCP' ]
-                go: [ '1.20', '1.19' ]
-        name: ${{ matrix.cloud }} Go ${{ matrix.go }} on Mac
-        steps:
-            - uses: actions/checkout@v1
-            - name: Setup go
-              uses: actions/setup-go@v2
-              with:
-                  go-version: ${{ matrix.go }}
-            - name: Format, Lint
-              shell: bash
-              run: ./ci/build.sh
-            - name: Test
-              shell: bash
-              env:
-                PARAMETERS_SECRET: ${{ secrets.PARAMETERS_SECRET }}
-                CLOUD_PROVIDER: ${{ matrix.cloud }}
-              run: ./ci/test.sh
-            - name: Upload coverage to Codecov
-              uses: codecov/codecov-action@v3
-              with:
-                token: ${{ secrets.CODE_COV_UPLOAD_TOKEN }}
-    build-test-windows:
-        runs-on: windows-latest
-        strategy:
-            fail-fast: false
-            matrix:
-                cloud: [ 'AWS', 'AZURE', 'GCP' ]
-                go: [ '1.20', '1.19' ]
-        name: ${{ matrix.cloud }} Go ${{ matrix.go }} on Windows
-        steps:
-            - uses: actions/checkout@v1
-            - name: Setup go
-              uses: actions/setup-go@v2
-              with:
-                  go-version: ${{ matrix.go }}
-            - name: Format, Lint
-              shell: cmd
-              run: ci\\build.bat
-            - uses: actions/setup-python@v1
-              with:
-                python-version: '3.x'
-                architecture: 'x64'
-            - name: Test
-              shell: cmd
-              env:
-                PARAMETERS_SECRET: ${{ secrets.PARAMETERS_SECRET }}
-                CLOUD_PROVIDER: ${{ matrix.cloud }}
-              run: ci\\test.bat
-            - name: Upload coverage to Codecov
-              uses: codecov/codecov-action@v3
-              with:
-                token: ${{ secrets.CODE_COV_UPLOAD_TOKEN }}
+  build-test-linux:
+    environment: gosnowflake-ci-env
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        # TODO(SIG-12289): re-enable tests on cloud providers other than AWS, and for v1.18
+        cloud: ["AWS"]
+        go: ['1.19']
+    name: ${{ matrix.cloud }} Go ${{ matrix.go }} on Ubuntu
+    steps:
+      - uses: actions/checkout@v1
+      - name: Setup go
+        uses: actions/setup-go@v2
+        with:
+          go-version: ${{ matrix.go }}
+      - name: Format, Lint
+        shell: bash
+        run: ./ci/build.sh
+      - name: Test
+        shell: bash
+        env:
+          PARAMETERS_SECRET: ${{ secrets.PARAMETERS_SECRET }}
+          CLOUD_PROVIDER: ${{ matrix.cloud }}
+        run: ./ci/test.sh

README.md

@@ -77,6 +77,9 @@ Set the Snowflake connection info in ``parameters.json``:
         "SNOWFLAKE_TEST_ROLE":      "<your_role>"
     }
 }
+
+You can find the complete file in the [Sigma 1Password](https://my.1password.com/vaults/likk64vc3hl7iaozanwj3dn7vu/allitems/72eslwc2yrglsfadkepljc45ai)
+
 ```
 
 Install [jq](https://stedolan.github.io/jq) so that the parameters can get parsed correctly, and run ``make test`` in your Go development environment:
@@ -110,3 +113,33 @@ You may use your preferred editor to edit the driver code. Make certain to run `
 For official support, contact Snowflake support at:
 [https://support.snowflake.net/](https://support.snowflake.net/).
 
+## Setting up the CI credentials
+
+You shouldn't need to do this, but in case we need to rotate the CI credentials, here are the steps I followed to create them:
+
+1. Install ``gpg`` if you don't already have it:
+
+```
+brew install gpg
+```
+
+2. Get the `gpg passphrase <https://my.1password.com/vaults/likk64vc3hl7iaozanwj3dn7vu/allitems/esdnmyqh5c3cze3k67tqrkd5s4>`_ and the raw `parameters.json file <https://my.1password.com/vaults/likk64vc3hl7iaozanwj3dn7vu/allitems/72eslwc2yrglsfadkepljc45ai>`_ from the Sigma 1Password.
+
+3. Use ``gpg``'s symmetric encryption mode to encrypt the ``parameters.json`` file. You'll be prompted twice to enter the passphrase:
+
+```
+gpg --symmetric --cipher-algo AES256 --output .github/workflows/parameters_aws_golang.json.gpg parameters.json
+```
+
+4. Get the `TEST_USER private key <https://sigmacomputing.1password.com/vaults/likk64vc3hl7iaozanwj3dn7vu/allitems/7g4gv6wjbbh6bgt7t6v6dlbhke>`_ from the Sigma 1Password. The TEST_USER keypair secret includes a public key, an encrypted private key, and the passphrase used to encrypt the private key; copy only the encrypted private key into ``rsa-2048-private-key-enc.p8``)
+
+5. Remove the passphrase from the private key (you'll be prompted for the private key passphrase), then use ``gpg``'s symmetric encryption mode to encrypt the resulting unencrypted private key (we only need one layer of encryption and it's easier to standardize on ``gpg``). As with the ``parameters.json`` file, you'll be prompted twice to enter the gpg passphrase:
+
+```
+openssl pkcs8 -in rsa-2048-private-key-enc.p8 -out rsa-2048-private-key.p8
+gpg --symmetric --cipher-algo AES256 --output .github/workflows/rsa-2048-private-key.p8.gpg rsa-2048-private-key.p8
+```
+
+6. Ensure that the gpg passphrase is configured properly in the `GitHub Environment <https://docs.github.com/en/actions/reference/environments>`_
+
+

arrow_chunk.go

@@ -5,6 +5,9 @@ package gosnowflake
 import (
 	"bytes"
 	"encoding/base64"
+	"io"
+	"strconv"
+	"strings"
 	"time"
 
 	"github.com/apache/arrow/go/v12/arrow"
@@ -67,6 +70,62 @@ func (arc *arrowResultChunk) decodeArrowBatch(scd *snowflakeChunkDownloader) (*[
 	return &records, arc.reader.Err()
 }
 
+// Note(Qing): Previously, the gosnowflake driver decodes the raw arrow chunks fetched from snowflake by
+// calling the decodeArrowBatch() function above. Instead of decoding here, we directly pass the raw records
+// to evaluator, along with neccesary metadata needed.
+func (arc *arrowResultChunk) passRawArrowBatch(scd *snowflakeChunkDownloader) (*[]arrow.Record, error) {
+	var records []arrow.Record
+
+	for {
+		rawRecord, err := arc.reader.Read()
+		if err == io.EOF {
+			break
+		} else if err != nil {
+			return nil, err
+		}
+		// Here we check all metadata from snowflake are preserved, so evaluator can decode accordingly
+		for idx, field := range rawRecord.Schema().Fields() {
+			// NOTE(Qing): Sometimes we see the rowtype metadata specify nullable as false but then still
+			// reveive nullable arrow records. Given that, we do not check the nullability here. Also, no
+			// need to compare names.
+			if !checkMetadata(field.Metadata, scd.RowSet.RowType[idx]) {
+				logger.Error("Lack or mismatch of necessary metadata to decode fetched raw arrow records")
+				return nil, &SnowflakeError{
+					Message: "Lack or mismatch of necessary metadata to decode fetched raw arrow records",
+				}
+			}
+		}
+		rawRecord.Retain()
+		records = append(records, rawRecord)
+	}
+	return &records, nil
+}
+
+func checkMetadata(actual arrow.Metadata, expected execResponseRowType) bool {
+	// LogicalType seems to be the only REALLY necessary metadata.
+	var hasLogicalType bool
+
+	for idx, key := range actual.Keys() {
+		switch strings.ToUpper(key) {
+		case "LOGICALTYPE":
+			hasLogicalType = true
+			if !strings.EqualFold(actual.Values()[idx], expected.Type) {
+				return false
+			}
+		case "SCALE":
+			switch strings.ToUpper(expected.Type) {
+			case "FIXED", "TIME", "TIMESTAMP_LTZ", "TIMESTAMP_NTZ":
+				if i64, err := strconv.ParseInt(actual.Values()[idx], 10, 64); err != nil || i64 != expected.Scale {
+					return false
+				}
+			default:
+			}
+		default:
+		}
+	}
+	return hasLogicalType
+}
+
 // Build arrow chunk based on RowSet of base64
 func buildFirstArrowChunk(rowsetBase64 string, loc *time.Location, alloc memory.Allocator) arrowResultChunk {
 	rowSetBytes, err := base64.StdEncoding.DecodeString(rowsetBase64)