From c30109df3c6dc77c09399062f231c5c2073d64c1 Mon Sep 17 00:00:00 2001 From: Tristen Harr Date: Tue, 5 Nov 2024 13:04:53 -0600 Subject: [PATCH 1/3] update to fix timestamp and bigint/hugeint filters --- ndc-duckduckapi/src/constants.ts | 55 ++++++++++-- ndc-duckduckapi/src/handlers/query.ts | 119 +++++++++++++++++++++----- 2 files changed, 147 insertions(+), 27 deletions(-) diff --git a/ndc-duckduckapi/src/constants.ts b/ndc-duckduckapi/src/constants.ts index 900403c..f48e874 100644 --- a/ndc-duckduckapi/src/constants.ts +++ b/ndc-duckduckapi/src/constants.ts @@ -38,39 +38,82 @@ export const SCALAR_TYPES: { [key: string]: ScalarType } = { type: "custom", argument_type: { type: "named", - name: "Int", + name: "BigInt", }, }, _lt: { type: "custom", argument_type: { type: "named", - name: "Int", + name: "BigInt", }, }, _gte: { type: "custom", argument_type: { type: "named", - name: "Int", + name: "BigInt", }, }, _lte: { type: "custom", argument_type: { type: "named", - name: "Int", + name: "BigInt", }, }, _neq: { type: "custom", argument_type: { type: "named", - name: "Int", + name: "BigInt", }, }, }, - }, Int: { + }, + UBigInt: { + representation: { + type: "biginteger" + }, + aggregate_functions: {}, + comparison_operators: { + _eq: { type: "equal" }, + _gt: { type: "custom", argument_type: { type: "named", name: "UBigInt" }}, + _lt: { type: "custom", argument_type: { type: "named", name: "UBigInt" }}, + _gte: { type: "custom", argument_type: { type: "named", name: "UBigInt" }}, + _lte: { type: "custom", argument_type: { type: "named", name: "UBigInt" }}, + _neq: { type: "custom", argument_type: { type: "named", name: "UBigInt" }}, + }, + }, + HugeInt: { + representation: { + type: "biginteger" + }, + aggregate_functions: {}, + comparison_operators: { + _eq: { type: "equal" }, + _gt: { type: "custom", argument_type: { type: "named", name: "HugeInt" }}, + _lt: { type: "custom", argument_type: { type: "named", name: "HugeInt" }}, + _gte: { type: "custom", argument_type: { type: "named", name: "HugeInt" }}, + _lte: { type: "custom", argument_type: { type: "named", name: "HugeInt" }}, + _neq: { type: "custom", argument_type: { type: "named", name: "HugeInt" }}, + }, + }, + UHugeInt: { + representation: { + type: "biginteger" + }, + aggregate_functions: {}, + comparison_operators: { + _eq: { type: "equal" }, + _gt: { type: "custom", argument_type: { type: "named", name: "UHugeInt" }}, + _lt: { type: "custom", argument_type: { type: "named", name: "UHugeInt" }}, + _gte: { type: "custom", argument_type: { type: "named", name: "UHugeInt" }}, + _lte: { type: "custom", argument_type: { type: "named", name: "UHugeInt" }}, + _neq: { type: "custom", argument_type: { type: "named", name: "UHugeInt" }}, + }, + }, + Int: { aggregate_functions: { // sum: { // result_type: { diff --git a/ndc-duckduckapi/src/handlers/query.ts b/ndc-duckduckapi/src/handlers/query.ts index a6d3245..2a36664 100644 --- a/ndc-duckduckapi/src/handlers/query.ts +++ b/ndc-duckduckapi/src/handlers/query.ts @@ -87,6 +87,41 @@ function wrap_rows(s: string): string { `; } +function isTimestampType(field_def: any): boolean { + if (!field_def) return false; + + function checkType(type: any): boolean { + if (type.type === "nullable") { + return checkType(type.underlying_type); + } + return type.type === "named" && type.name === "Timestamp"; + } + + return checkType(field_def.type); +} + +function getIntegerType(field_def: any): string | null { + if (!field_def) return null; + + function checkType(type: any): string | null { + if (type.type === "nullable") { + return checkType(type.underlying_type); + } + if (type.type === "named") { + switch (type.name) { + case "BigInt": return "BIGINT"; + case "UBigInt": return "UBIGINT"; + case "HugeInt": return "HUGEINT"; + case "UHugeInt": return "UHUGEINT"; + default: return null; + } + } + return null; + } + + return checkType(field_def.type); +} + function build_where( expression: Expression, collection_relationships: { @@ -95,7 +130,9 @@ function build_where( args: any[], variables: QueryVariables, prefix: string, - collection_aliases: { [k: string]: string } + collection_aliases: { [k: string]: string }, + config: Configuration, + query_request: QueryRequest ): string { let sql = ""; switch (expression.type) { @@ -111,6 +148,10 @@ function build_where( } break; case "binary_comparison_operator": + const object_type = config.duckdbConfig?.object_types[query_request.collection]; + const field_def = object_type?.fields[expression.column.name]; + const isTimestamp = isTimestampType(field_def); + const integerType = getIntegerType(field_def); switch (expression.value.type) { case "scalar": args.push(expression.value.value); @@ -127,33 +168,57 @@ function build_where( } switch (expression.operator) { case "_eq": - sql = `${expression.column.name} = ?`; - break; - case "_like": - args[args.length - 1] = `%${args[args.length - 1]}%`; - sql = `${expression.column.name} LIKE ?`; - break; - case "_glob": - sql = `${expression.column.name} GLOB ?`; + sql = isTimestamp + ? `${escape_double(expression.column.name)} = CAST(? AS TIMESTAMP)` + : integerType + ? `${escape_double(expression.column.name)} = CAST(? AS ${integerType})` + : `${escape_double(expression.column.name)} = ?`; break; case "_neq": - sql = `${expression.column.name} != ?`; + sql = isTimestamp + ? `${escape_double(expression.column.name)} != CAST(? AS TIMESTAMP)` + : integerType + ? `${escape_double(expression.column.name)} != CAST(? AS ${integerType})` + : `${escape_double(expression.column.name)} != ?`; break; case "_gt": - sql = `${expression.column.name} > ?`; + sql = isTimestamp + ? `${escape_double(expression.column.name)} > CAST(? AS TIMESTAMP)` + : integerType + ? `${escape_double(expression.column.name)} > CAST(? AS ${integerType})` + : `${escape_double(expression.column.name)} > ?`; break; case "_lt": - sql = `${expression.column.name} < ?`; + sql = isTimestamp + ? `${escape_double(expression.column.name)} < CAST(? AS TIMESTAMP)` + : integerType + ? `${escape_double(expression.column.name)} < CAST(? AS ${integerType})` + : `${escape_double(expression.column.name)} < ?`; break; case "_gte": - sql = `${expression.column.name} >= ?`; + sql = isTimestamp + ? `${escape_double(expression.column.name)} >= CAST(? AS TIMESTAMP)` + : integerType + ? `${escape_double(expression.column.name)} >= CAST(? AS ${integerType})` + : `${escape_double(expression.column.name)} >= ?`; break; case "_lte": - sql = `${expression.column.name} <= ?`; + sql = isTimestamp + ? `${escape_double(expression.column.name)} <= CAST(? AS TIMESTAMP)` + : integerType + ? `${escape_double(expression.column.name)} <= CAST(? AS ${integerType})` + : `${escape_double(expression.column.name)} <= ?`; + break; + case "_like": + args[args.length - 1] = `%${args[args.length - 1]}%`; + sql = `${escape_double(expression.column.name)} LIKE ?`; + break; + case "_glob": + sql = `${escape_double(expression.column.name)} GLOB ?`; break; default: throw new Forbidden( - "Binary Comparison Custom Operator not implemented", + `Binary Comparison Custom Operator ${expression.operator} not implemented`, {} ); } @@ -170,7 +235,9 @@ function build_where( args, variables, prefix, - collection_aliases + collection_aliases, + config, + query_request ); clauses.push(res); } @@ -189,7 +256,9 @@ function build_where( args, variables, prefix, - collection_aliases + collection_aliases, + config, + query_request ); clauses.push(res); } @@ -203,7 +272,9 @@ function build_where( args, variables, prefix, - collection_aliases + collection_aliases, + config, + query_request ); sql = `NOT (${not_result})`; break; @@ -228,7 +299,9 @@ function build_where( args, variables, prefix, - collection_aliases + collection_aliases, + config, + query_request ) : "1 = 1" } @@ -366,7 +439,9 @@ function build_query( agg_args, variables, collection_alias, - config.duckdbConfig.collection_aliases + config.duckdbConfig.collection_aliases, + config, + query_request )})` ); } @@ -455,7 +530,9 @@ function build_query( args, variables, collection_alias, - config.duckdbConfig.collection_aliases + config.duckdbConfig.collection_aliases, + config, + query_request )})` ); } From a06845e889a54a563a305b9b5f34712395216fe2 Mon Sep 17 00:00:00 2001 From: Tristen Harr Date: Tue, 5 Nov 2024 13:12:33 -0600 Subject: [PATCH 2/3] fix generation code --- ndc-duckduckapi/src/generate-config.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ndc-duckduckapi/src/generate-config.ts b/ndc-duckduckapi/src/generate-config.ts index 31b6b8b..a78766c 100644 --- a/ndc-duckduckapi/src/generate-config.ts +++ b/ndc-duckduckapi/src/generate-config.ts @@ -30,7 +30,7 @@ const determineType = (t: string): string => { case "DOUBLE": return "Float"; case "HUGEINT": - return "BigInt"; + return "HugeInt"; case "INTEGER": return "Int"; case "INTERVAL": @@ -50,9 +50,9 @@ const determineType = (t: string): string => { case "TINYINT": return "Int"; case "UBIGINT": - return "BigInt"; + return "UBigInt"; case "UHUGEINT": - return "BigInt"; + return "UHugeInt"; case "UINTEGER": return "Int"; case "USMALLINT": From 504b19f305daa0e98026b414534a50e71cff4846 Mon Sep 17 00:00:00 2001 From: Tristen Harr Date: Tue, 5 Nov 2024 13:30:01 -0600 Subject: [PATCH 3/3] extract LHS and RHS for better code re-use --- ndc-duckduckapi/src/handlers/query.ts | 48 +++++++++------------------ 1 file changed, 16 insertions(+), 32 deletions(-) diff --git a/ndc-duckduckapi/src/handlers/query.ts b/ndc-duckduckapi/src/handlers/query.ts index 2a36664..8746a22 100644 --- a/ndc-duckduckapi/src/handlers/query.ts +++ b/ndc-duckduckapi/src/handlers/query.ts @@ -122,6 +122,11 @@ function getIntegerType(field_def: any): string | null { return checkType(field_def.type); } +function getRhsExpression(type: string | null): string { + if (!type) return "?"; + return `CAST(? AS ${type})`; +} + function build_where( expression: Expression, collection_relationships: { @@ -152,6 +157,9 @@ function build_where( const field_def = object_type?.fields[expression.column.name]; const isTimestamp = isTimestampType(field_def); const integerType = getIntegerType(field_def); + const type = isTimestamp ? "TIMESTAMP" : integerType; + const lhs = escape_double(expression.column.name); + const rhs = getRhsExpression(type); switch (expression.value.type) { case "scalar": args.push(expression.value.value); @@ -168,53 +176,29 @@ function build_where( } switch (expression.operator) { case "_eq": - sql = isTimestamp - ? `${escape_double(expression.column.name)} = CAST(? AS TIMESTAMP)` - : integerType - ? `${escape_double(expression.column.name)} = CAST(? AS ${integerType})` - : `${escape_double(expression.column.name)} = ?`; + sql = `${lhs} = ${rhs}`; break; case "_neq": - sql = isTimestamp - ? `${escape_double(expression.column.name)} != CAST(? AS TIMESTAMP)` - : integerType - ? `${escape_double(expression.column.name)} != CAST(? AS ${integerType})` - : `${escape_double(expression.column.name)} != ?`; + sql = `${lhs} != ${rhs}`; break; case "_gt": - sql = isTimestamp - ? `${escape_double(expression.column.name)} > CAST(? AS TIMESTAMP)` - : integerType - ? `${escape_double(expression.column.name)} > CAST(? AS ${integerType})` - : `${escape_double(expression.column.name)} > ?`; + sql = `${lhs} > ${rhs}`; break; case "_lt": - sql = isTimestamp - ? `${escape_double(expression.column.name)} < CAST(? AS TIMESTAMP)` - : integerType - ? `${escape_double(expression.column.name)} < CAST(? AS ${integerType})` - : `${escape_double(expression.column.name)} < ?`; + sql = `${lhs} < ${rhs}`; break; case "_gte": - sql = isTimestamp - ? `${escape_double(expression.column.name)} >= CAST(? AS TIMESTAMP)` - : integerType - ? `${escape_double(expression.column.name)} >= CAST(? AS ${integerType})` - : `${escape_double(expression.column.name)} >= ?`; + sql = `${lhs} >= ${rhs}`; break; case "_lte": - sql = isTimestamp - ? `${escape_double(expression.column.name)} <= CAST(? AS TIMESTAMP)` - : integerType - ? `${escape_double(expression.column.name)} <= CAST(? AS ${integerType})` - : `${escape_double(expression.column.name)} <= ?`; + sql = `${lhs} <= ${rhs}`; break; case "_like": args[args.length - 1] = `%${args[args.length - 1]}%`; - sql = `${escape_double(expression.column.name)} LIKE ?`; + sql = `${lhs} LIKE ?`; break; case "_glob": - sql = `${escape_double(expression.column.name)} GLOB ?`; + sql = `${lhs} GLOB ?`; break; default: throw new Forbidden(