Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

SNOW-911181: Add parameter to disable query context cache #912

Merged
merged 1 commit into from
Sep 11, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 14 additions & 1 deletion connection.go
Original file line number Diff line number Diff line change
Expand Up @@ -141,7 +141,14 @@ func (sc *snowflakeConn) exec(
return nil, err
}

sc.queryContextCache.add(sc, data.Data.QueryContext.Entries...)
if !sc.cfg.DisableQueryContextCache && data.Data.QueryContext != nil {
queryContext, err := extractQueryContext(data)
if err != nil {
logger.Errorf("error while decoding query context: ", err)
} else {
sc.queryContextCache.add(sc, queryContext.Entries...)
}
}

// handle PUT/GET commands
if isFileTransfer(query) {
Expand All @@ -160,6 +167,12 @@ func (sc *snowflakeConn) exec(
return data, err
}

func extractQueryContext(data *execResponse) (queryContext, error) {
var queryContext queryContext
err := json.Unmarshal(data.Data.QueryContext, &queryContext)
return queryContext, err
}

func (sc *snowflakeConn) Begin() (driver.Tx, error) {
return sc.BeginTx(sc.ctx, driver.TxOptions{})
}
Expand Down
3 changes: 3 additions & 0 deletions doc.go
Original file line number Diff line number Diff line change
Expand Up @@ -118,6 +118,9 @@
- tracing: Specifies the logging level to be used. Set to error by default.
Valid values are trace, debug, info, print, warning, error, fatal, panic.

- disableQueryContextCache: disables parsing of query context returned from server and resending it to server as well.
sfc-gh-dprzybysz marked this conversation as resolved.
Show resolved Hide resolved
Default value is false.

All other parameters are interpreted as session parameters (https://docs.snowflake.com/en/sql-reference/parameters.html).
For example, the TIMESTAMP_OUTPUT_FORMAT session parameter can be set by adding:

Expand Down Expand Up @@ -931,7 +934,7 @@
strings.ReplaceAll(fname, "\\", "\\\\"),
tableName)
dbt.mustExecContext(WithFileStream(context.Background(), fileStream),
sqlText)

Check failure on line 937 in doc.go

View workflow job for this annotation

GitHub Actions / AWS Go 1.20 on Windows

package comment is detached; there should be no blank lines between it and the package statement

Check failure on line 937 in doc.go

View workflow job for this annotation

GitHub Actions / AWS Go 1.19 on Windows

package comment is detached; there should be no blank lines between it and the package statement

Check failure on line 937 in doc.go

View workflow job for this annotation

GitHub Actions / AZURE Go 1.20 on Windows

package comment is detached; there should be no blank lines between it and the package statement

Check failure on line 937 in doc.go

View workflow job for this annotation

GitHub Actions / AZURE Go 1.19 on Windows

package comment is detached; there should be no blank lines between it and the package statement

Check failure on line 937 in doc.go

View workflow job for this annotation

GitHub Actions / GCP Go 1.20 on Windows

package comment is detached; there should be no blank lines between it and the package statement

Check failure on line 937 in doc.go

View workflow job for this annotation

GitHub Actions / GCP Go 1.19 on Windows

package comment is detached; there should be no blank lines between it and the package statement

Note: PUT statements are not supported for multi-statement queries.

Expand Down
12 changes: 12 additions & 0 deletions dsn.go
Original file line number Diff line number Diff line change
Expand Up @@ -97,6 +97,8 @@ type Config struct {
IDToken string // Internally used to cache the Id Token for external browser
ClientRequestMfaToken ConfigBool // When true the MFA token is cached in the credential manager. True by default in Windows/OSX. False for Linux.
ClientStoreTemporaryCredential ConfigBool // When true the ID token is cached in the credential manager. True by default in Windows/OSX. False for Linux.

DisableQueryContextCache bool // Should HTAP query context cache be disabled
}

// Validate enables testing if config is correct.
Expand Down Expand Up @@ -230,6 +232,9 @@ func DSN(cfg *Config) (dsn string, err error) {
if cfg.TmpDirPath != "" {
params.Add("tmpDirPath", cfg.TmpDirPath)
}
if cfg.DisableQueryContextCache {
params.Add("disableQueryContextCache", "true")
}

params.Add("ocspFailOpen", strconv.FormatBool(cfg.OCSPFailOpen != OCSPFailOpenFalse))

Expand Down Expand Up @@ -702,6 +707,13 @@ func parseDSNParams(cfg *Config, params string) (err error) {
cfg.Tracing = value
case "tmpDirPath":
cfg.TmpDirPath = value
case "disableQueryContextCache":
var b bool
b, err = strconv.ParseBool(value)
if err != nil {
return
sfc-gh-dheyman marked this conversation as resolved.
Show resolved Hide resolved
}
cfg.DisableQueryContextCache = b
default:
if cfg.Params == nil {
cfg.Params = make(map[string]*string)
Expand Down
33 changes: 30 additions & 3 deletions dsn_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -572,9 +572,10 @@ func TestParseDSN(t *testing.T) {
Account: "a", User: "u", Password: "p",
Protocol: "https", Host: "a.r.c.snowflakecomputing.com", Port: 443,
Database: "db", Schema: "s", ValidateDefaultParameters: ConfigBoolTrue, OCSPFailOpen: OCSPFailOpenTrue,
ClientTimeout: 300 * time.Second,
JWTClientTimeout: 45 * time.Second,
ExternalBrowserTimeout: defaultExternalBrowserTimeout,
ClientTimeout: 300 * time.Second,
JWTClientTimeout: 45 * time.Second,
ExternalBrowserTimeout: defaultExternalBrowserTimeout,
DisableQueryContextCache: false,
},
ocspMode: ocspModeFailOpen,
err: nil,
Expand All @@ -593,6 +594,20 @@ func TestParseDSN(t *testing.T) {
ocspMode: ocspModeFailOpen,
err: nil,
},
{
dsn: "u:[email protected]/db/s?account=a.r.c&disableQueryContextCache=true",
config: &Config{
Account: "a", User: "u", Password: "p",
Protocol: "https", Host: "a.r.c.snowflakecomputing.com", Port: 443,
Database: "db", Schema: "s", ValidateDefaultParameters: ConfigBoolTrue, OCSPFailOpen: OCSPFailOpenTrue,
ClientTimeout: defaultClientTimeout,
JWTClientTimeout: defaultJWTClientTimeout,
ExternalBrowserTimeout: defaultExternalBrowserTimeout,
DisableQueryContextCache: true,
sfc-gh-dprzybysz marked this conversation as resolved.
Show resolved Hide resolved
},
ocspMode: ocspModeFailOpen,
err: nil,
},
}

for _, at := range []AuthType{AuthTypeExternalBrowser, AuthTypeOAuth} {
Expand Down Expand Up @@ -743,6 +758,9 @@ func TestParseDSN(t *testing.T) {
if test.config.TmpDirPath != cfg.TmpDirPath {
t.Fatalf("%v: Failed to match TmpDirPatch. expected: %v, got: %v", i, test.config.TmpDirPath, cfg.TmpDirPath)
}
if test.config.DisableQueryContextCache != cfg.DisableQueryContextCache {
t.Fatalf("%v: Failed to match DisableQueryContextCache. expected: %v, got: %v", i, test.config.DisableQueryContextCache, cfg.DisableQueryContextCache)
}
case test.err != nil:
driverErrE, okE := test.err.(*SnowflakeError)
driverErrG, okG := err.(*SnowflakeError)
Expand Down Expand Up @@ -1133,6 +1151,15 @@ func TestDSN(t *testing.T) {
},
dsn: "u:[email protected]:443?ocspFailOpen=true&region=b.c&tmpDirPath=%2Ftmp&validateDefaultParameters=true",
},
{
cfg: &Config{
User: "u",
Password: "p",
Account: "a.b.c",
DisableQueryContextCache: true,
sfc-gh-dprzybysz marked this conversation as resolved.
Show resolved Hide resolved
},
dsn: "u:[email protected]:443?disableQueryContextCache=true&ocspFailOpen=true&region=b.c&validateDefaultParameters=true",
},
}
for _, test := range testcases {
t.Run(test.dsn, func(t *testing.T) {
Expand Down
4 changes: 4 additions & 0 deletions htap.go
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,10 @@ const (
defaultQueryContextCacheSize = 5
)

type queryContext struct {
Entries []queryContextEntry `json:"entries,omitempty"`
}

type queryContextEntry struct {
ID int `json:"id"`
Timestamp int64 `json:"timestamp"`
Expand Down
14 changes: 14 additions & 0 deletions htap_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -412,3 +412,17 @@ func htapTestSnowflakeConn() *snowflakeConn {
},
}
}

func TestQueryContextCacheDisabled(t *testing.T) {
origDsn := dsn
defer func() {
dsn = origDsn
}()
dsn += "&disableQueryContextCache=true"
runSnowflakeConnTest(t, func(sct *SCTest) {
sct.mustExec("SELECT 1", nil)
if len(sct.sc.queryContextCache.entries) > 0 {
t.Error("should not contain any entries")
}
})
}
5 changes: 2 additions & 3 deletions query.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
package gosnowflake

import (
"encoding/json"
"time"
)

Expand Down Expand Up @@ -120,9 +121,7 @@ type execResponseData struct {
Operation string `json:"operation,omitempty"`

// HTAP
QueryContext struct {
Entries []queryContextEntry `json:"entries,omitempty"`
} `json:"queryContext,omitempty"`
QueryContext json.RawMessage `json:"queryContext,omitempty"`
}

type execResponse struct {
Expand Down
Loading