diff --git a/clients/databricks/store.go b/clients/databricks/store.go index 02df68b94..13d2b1618 100644 --- a/clients/databricks/store.go +++ b/clients/databricks/store.go @@ -7,6 +7,7 @@ import ( "log/slog" "os" "path/filepath" + "strings" _ "github.com/databricks/databricks-sql-go" "github.com/databricks/databricks-sql-go/driverctx" @@ -120,9 +121,38 @@ func (s Store) PrepareTemporaryTable(ctx context.Context, tableData *optimizatio } }() + var ordinalColumns []string + for idx, column := range tableData.ReadOnlyInMemoryCols().ValidColumns() { + ordinalColumn := fmt.Sprintf("_c%d", idx) + switch column.KindDetails.Kind { + case typing.Array.Kind: + ordinalColumn = fmt.Sprintf(`PARSE_JSON(%s)`, ordinalColumn) + } + + ordinalColumns = append(ordinalColumns, ordinalColumn) + } + // Copy file from DBFS -> table via COPY INTO, ref: https://docs.databricks.com/en/sql/language-manual/delta-copy-into.html // We'll need \\\\N here because we need to string escape. - copyCommand := fmt.Sprintf(`COPY INTO %s BY POSITION FROM '%s' FILEFORMAT = CSV FORMAT_OPTIONS ('escape' = '"', 'delimiter' = '\t', 'header' = 'false', 'nullValue' = '\\\\N')`, tempTableID.FullyQualifiedName(), file.DBFSFilePath()) + copyCommand := fmt.Sprintf(` +COPY INTO %s +BY POSITION +FROM ( + SELECT %s FROM '%s' +) +FILEFORMAT = CSV +FORMAT_OPTIONS ( + 'escape' = '"', + 'delimiter' = '\t', + 'header' = 'false', + 'nullValue' = '\\\\N' +);`, + // COPY INTO + tempTableID.FullyQualifiedName(), + // SELECT columns FROM file + strings.Join(ordinalColumns, ", "), file.DBFSFilePath(), + ) + if _, err = s.ExecContext(ctx, copyCommand); err != nil { return fmt.Errorf("failed to run COPY INTO for temporary table: %w", err) } diff --git a/lib/typing/converters/string_converter.go b/lib/typing/converters/string_converter.go index 573276b92..ab4f70e70 100644 --- a/lib/typing/converters/string_converter.go +++ b/lib/typing/converters/string_converter.go @@ -5,6 +5,8 @@ import ( "fmt" "time" + "github.com/artie-labs/transfer/lib/config/constants" + "github.com/artie-labs/transfer/lib/typing" "github.com/artie-labs/transfer/lib/typing/ext" ) @@ -78,6 +80,14 @@ func (TimestampTZConverter) Convert(value any) (string, error) { type ArrayConverter struct{} func (ArrayConverter) Convert(value any) (string, error) { + // If the column value is TOASTED, we should return an array with the TOASTED placeholder + // We're doing this to make sure that the value matches the schema. + if stringValue, ok := value.(string); ok { + if stringValue == constants.ToastUnavailableValuePlaceholder { + return fmt.Sprintf(`["%s"]`, constants.ToastUnavailableValuePlaceholder), nil + } + } + colValBytes, err := json.Marshal(value) if err != nil { return "", err diff --git a/lib/typing/converters/string_converter_test.go b/lib/typing/converters/string_converter_test.go new file mode 100644 index 000000000..62f046cba --- /dev/null +++ b/lib/typing/converters/string_converter_test.go @@ -0,0 +1,24 @@ +package converters + +import ( + "testing" + + "github.com/artie-labs/transfer/lib/config/constants" + "github.com/stretchr/testify/assert" +) + +func TestArrayConverter(t *testing.T) { + // Array + { + // Normal arrays + val, err := ArrayConverter{}.Convert([]string{"foo", "bar"}) + assert.NoError(t, err) + assert.Equal(t, `["foo","bar"]`, val) + } + { + // Toasted array + val, err := ArrayConverter{}.Convert(constants.ToastUnavailableValuePlaceholder) + assert.NoError(t, err) + assert.Equal(t, `["__debezium_unavailable_value"]`, val) + } +} diff --git a/lib/typing/values/string_test.go b/lib/typing/values/string_test.go index 8db4751c4..5ad48a2af 100644 --- a/lib/typing/values/string_test.go +++ b/lib/typing/values/string_test.go @@ -172,9 +172,18 @@ func TestToString(t *testing.T) { } { // Array - val, err := ToString([]string{"foo", "bar"}, typing.Array) - assert.NoError(t, err) - assert.Equal(t, `["foo","bar"]`, val) + { + // Normal arrays + val, err := ToString([]string{"foo", "bar"}, typing.Array) + assert.NoError(t, err) + assert.Equal(t, `["foo","bar"]`, val) + } + { + // Toasted array + val, err := ToString(constants.ToastUnavailableValuePlaceholder, typing.Array) + assert.NoError(t, err) + assert.Equal(t, `["__debezium_unavailable_value"]`, val) + } } { // Integer column