From 9755eb7037455b005d9de122adb975d6157f7409 Mon Sep 17 00:00:00 2001 From: github-actions Date: Wed, 13 Mar 2024 08:23:03 +0000 Subject: [PATCH] update json schemas --- schemas/job.schema.json | 1158 +++++++++++++++++++-------------------- 1 file changed, 579 insertions(+), 579 deletions(-) diff --git a/schemas/job.schema.json b/schemas/job.schema.json index 683bd708..36dac71a 100644 --- a/schemas/job.schema.json +++ b/schemas/job.schema.json @@ -19,31 +19,116 @@ "properties": { "uses": { "enum": [ + "map", + "filter", "http.write", "http.receiver", - "filter", - "std.write", - "std.read", - "jinja_template", - "rename_field", - "sequence", - "add_field", "redis.write", "redis.lookup", "redis.read_stream", "remove_field", "azure.read_event_hub", + "std.write", + "std.read", + "jinja_template", "relational.write", "relational.read", - "map", + "rename_field", + "add_field", + "sequence", + "files.read_csv", "parquet.write", "parquet.read", - "cassandra.write", - "files.read_csv" + "cassandra.write" ] } }, "allOf": [ + { + "if": { + "properties": { + "uses": { + "description": "Block type", + "type": "string", + "const": "map" + } + }, + "required": ["uses"] + }, + "then": { + "properties": { + "with": { + "title": "map", + "description": "Maps a record into a new output based on expressions", + "type": "object", + "properties": { + "expression": { + "description": "Expression", + "type": ["object", "string"] + }, + "language": { + "description": "Language", + "type": "string", + "enum": ["jmespath", "sql"] + } + }, + "additionalProperties": false, + "required": ["expression", "language"], + "examples": [ + { + "expression": { + "first_name": "first_name", + "last_name": "last_name", + "greeting": "'Hello ' || CASE WHEN gender = 'F' THEN 'Ms.' WHEN gender = 'M' THEN 'Mr.' ELSE 'N/A' END || ' ' || full_name", + "country": "country", + "full_name": "full_name" + }, + "language": "sql" + }, + { + "expression": "{\"CustomerId\": \"customer_id\", \"FirstName\": \"first_name\", \"LastName\": \"last_name\", \"Company\": \"company\", \"Location\": {\"Street\": \"address\", \"City\": \"city\", \"State\": \"state\", \"Country\": \"country\", \"PostalCode\": \"postal_code\"}, \"Phone\": \"phone\", \"Fax\": \"fax\", \"Email\": \"email\"}", + "language": "jmespath" + } + ] + } + } + } + }, + { + "if": { + "properties": { + "uses": { + "description": "Block type", + "type": "string", + "const": "filter" + } + }, + "required": ["uses"] + }, + "then": { + "properties": { + "with": { + "title": "filter", + "description": "Filter records", + "type": "object", + "properties": { + "expression": { + "description": "Expression", + "type": "string" + }, + "language": { + "description": "Language", + "type": "string", + "enum": ["jmespath", "sql"] + } + }, + "additionalProperties": false, + "required": ["expression", "language"], + "examples": [{ "language": "sql", "expression": "age>20" }] + } + } + } + }, { "if": { "properties": { @@ -290,7 +375,7 @@ "uses": { "description": "Block type", "type": "string", - "const": "filter" + "const": "redis.write" } }, "required": ["uses"] @@ -298,43 +383,48 @@ "then": { "properties": { "with": { - "title": "filter", - "description": "Filter records", + "title": "redis.write", + "description": "Write to a Redis data structure", "type": "object", "properties": { - "expression": { - "description": "Expression", + "connection": { + "title": "Connection name", "type": "string" }, - "language": { - "description": "Language", + "command": { + "enum": [ + "HSET", + "SADD", + "XADD", + "RPUSH", + "LPUSH", + "SET", + "ZADD" + ], + "default": "HSET", "type": "string", - "enum": ["jmespath", "sql"] + "title": "Redis command", + "description": "Redis command" + }, + "key": { + "description": "Field to use as the Redis key", + "type": "object", + "properties": { + "expression": { + "description": "Expression", + "type": "string" + }, + "language": { + "description": "Language", + "type": "string", + "enum": ["jmespath", "sql"] + } + }, + "required": ["expression", "language"] } }, "additionalProperties": false, - "required": ["expression", "language"], - "examples": [{ "language": "sql", "expression": "age>20" }] - } - } - } - }, - { - "if": { - "properties": { - "uses": { - "description": "Block type", - "type": "string", - "const": "std.write" - } - }, - "required": ["uses"] - }, - "then": { - "properties": { - "with": { - "title": "std.write", - "description": "Write to the standard output" + "required": ["connection", "key"] } } } @@ -345,7 +435,7 @@ "uses": { "description": "Block type", "type": "string", - "const": "std.read" + "const": "redis.lookup" } }, "required": ["uses"] @@ -353,8 +443,38 @@ "then": { "properties": { "with": { - "title": "std.read", - "description": "Read from the standard input" + "title": "redis.lookup", + "description": "Lookup data from Redis using the given command and key", + "type": "object", + "properties": { + "connection": { + "title": "Connection name", + "type": "string" + }, + "cmd": { + "title": "Redis command", + "description": "The command to execute", + "type": "string" + }, + "args": { + "title": "Redis command arguments", + "description": "The list of expressions produces arguments", + "type": "array", + "items": { "type": "string" } + }, + "language": { + "description": "Language", + "type": "string", + "enum": ["jmespath", "sql"] + }, + "field": { + "type": "string", + "title": "Target field", + "description": "The field to write the result to" + } + }, + "additionalProperties": false, + "required": ["connection", "cmd", "args", "language", "field"] } } } @@ -365,7 +485,7 @@ "uses": { "description": "Block type", "type": "string", - "const": "jinja_template" + "const": "redis.read_stream" } }, "required": ["uses"] @@ -373,28 +493,28 @@ "then": { "properties": { "with": { - "title": "jinja_template", - "description": "Apply Jinja template to a field", + "title": "redis.read_stream", + "description": "Read from Redis stream", "type": "object", "properties": { - "field": { "description": "Field", "type": "string" }, - "template": { - "description": "Jinja Template", + "connection": { + "description": "Connection name", "type": "string" + }, + "stream_name": { + "type": "string", + "title": "Source stream name", + "description": "Source stream name" + }, + "snapshot": { + "type": "boolean", + "title": "Snapshot current entries and quit", + "description": "Snapshot current entries and quit", + "default": false } }, "additionalProperties": false, - "required": ["field", "template"], - "examples": [ - { - "field": "name.full_name", - "template": "{{ name.fname }} {{ name.lname }}" - }, - { - "field": "name.fname_upper", - "template": "{{ name.fname | upper }}" - } - ] + "required": ["connection", "stream_name"] } } } @@ -405,7 +525,7 @@ "uses": { "description": "Block type", "type": "string", - "const": "rename_field" + "const": "remove_field" } }, "required": ["uses"] @@ -413,12 +533,12 @@ "then": { "properties": { "with": { - "title": "rename_field", - "description": "Renames fields. All other fields remain unchanged", + "title": "remove_field", + "description": "Remove fields", "type": "object", "oneOf": [ { - "description": "Rename multiple fields", + "description": "Remove multiple fields", "properties": { "fields": { "type": "array", @@ -426,147 +546,13 @@ "items": { "type": "object", "properties": { - "from_field": { - "description": "From field", - "type": "string" - }, - "to_field": { - "description": "To field", + "field": { + "description": "Field", "type": "string" } }, "additionalProperties": false, - "required": ["from_field", "to_field"], - "examples": [ - { - "fields": [ - { - "from_field": "name.lname", - "to_field": "name.last_name" - }, - { - "from_field": "name.fname", - "to_field": "name.first_name" - } - ] - } - ] - } - } - }, - "required": ["fields"], - "additionalProperties": false - }, - { - "description": "Rename one field", - "properties": { - "from_field": { - "description": "From field", - "type": "string" - }, - "to_field": { - "description": "To field", - "type": "string" - } - }, - "additionalProperties": false, - "required": ["from_field", "to_field"], - "examples": [ - { - "from_field": "name.lname", - "to_field": "name.last_name" - } - ] - } - ] - } - } - } - }, - { - "if": { - "properties": { - "uses": { - "description": "Block type", - "type": "string", - "const": "sequence" - } - }, - "required": ["uses"] - }, - "then": { - "properties": { - "with": { - "title": "sequence", - "description": "Add a sequence number field to data", - "type": "object", - "additionalProperties": false, - "examples": [], - "required": [], - "properties": { - "field": { - "type": "string", - "title": "Name of new sequence field" - }, - "start": { - "type": "number", - "title": "Start entry", - "default": 1, - "examples": [] - }, - "increment": { - "type": "number", - "title": "Increment between sequences", - "examples": [] - } - } - } - } - } - }, - { - "if": { - "properties": { - "uses": { - "description": "Block type", - "type": "string", - "const": "add_field" - } - }, - "required": ["uses"] - }, - "then": { - "properties": { - "with": { - "title": "add_field", - "description": "Add fields to a record", - "type": "object", - "oneOf": [ - { - "description": "Add multiple fields", - "properties": { - "fields": { - "type": "array", - "description": "Fields", - "items": { - "type": "object", - "properties": { - "field": { - "description": "Field", - "type": "string" - }, - "expression": { - "description": "Expression", - "type": "string" - }, - "language": { - "description": "Language", - "type": "string", - "enum": ["jmespath", "sql"] - } - }, - "additionalProperties": false, - "required": ["field", "expression", "language"] + "required": ["field"] } } }, @@ -575,43 +561,20 @@ "examples": [ { "fields": [ - { - "field": "name.full_name", - "language": "jmespath", - "expression": "concat([name.fname, ' ', name.lname])" - }, - { - "field": "name.fname_upper", - "language": "jmespath", - "expression": "upper(name.fname)" - } + { "field": "credit_card" }, + { "field": "name.mname" } ] } ] }, { - "description": "Add one field", + "description": "Remove one field", "properties": { - "field": { "description": "Field", "type": "string" }, - "expression": { - "description": "Expression", - "type": "string" - }, - "language": { - "description": "Language", - "type": "string", - "enum": ["jmespath", "sql"] - } + "field": { "description": "Field", "type": "string" } }, "additionalProperties": false, - "required": ["field", "expression", "language"], - "examples": [ - { - "field": "country", - "language": "sql", - "expression": "country_code || ' - ' || UPPER(country_name)" - } - ] + "required": ["field"], + "examples": [{ "field": "credit_card" }] } ] } @@ -624,7 +587,7 @@ "uses": { "description": "Block type", "type": "string", - "const": "redis.write" + "const": "azure.read_event_hub" } }, "required": ["uses"] @@ -632,98 +595,43 @@ "then": { "properties": { "with": { - "title": "redis.write", - "description": "Write to a Redis data structure", + "title": "azure.read_event_hub", + "description": "Read from Azure Event Hub", "type": "object", "properties": { - "connection": { - "title": "Connection name", - "type": "string" - }, - "command": { - "enum": [ - "HSET", - "SADD", - "XADD", - "RPUSH", - "LPUSH", - "SET", - "ZADD" - ], - "default": "HSET", + "event_hub_connection_string": { "type": "string", - "title": "Redis command", - "description": "Redis command" - }, - "key": { - "description": "Field to use as the Redis key", - "type": "object", - "properties": { - "expression": { - "description": "Expression", - "type": "string" - }, - "language": { - "description": "Language", - "type": "string", - "enum": ["jmespath", "sql"] - } - }, - "required": ["expression", "language"] - } - }, - "additionalProperties": false, - "required": ["connection", "key"] - } - } - } - }, - { - "if": { - "properties": { - "uses": { - "description": "Block type", - "type": "string", - "const": "redis.lookup" - } - }, - "required": ["uses"] - }, - "then": { - "properties": { - "with": { - "title": "redis.lookup", - "description": "Lookup data from Redis using the given command and key", - "type": "object", - "properties": { - "connection": { - "title": "Connection name", - "type": "string" + "description": "The connection string for the Azure Event Hub namespace." }, - "cmd": { - "title": "Redis command", - "description": "The command to execute", - "type": "string" + "event_hub_consumer_group_name": { + "type": "string", + "description": "The name of the consumer group to read events from." }, - "args": { - "title": "Redis command arguments", - "description": "The list of expressions produces arguments", - "type": "array", - "items": { "type": "string" } + "event_hub_name": { + "type": "string", + "description": "The name of the Azure Event Hub." }, - "language": { - "description": "Language", + "checkpoint_store_connection_string": { "type": "string", - "enum": ["jmespath", "sql"] + "description": "The connection string for the Azure Storage account used as the checkpoint store." }, - "field": { + "checkpoint_store_container_name": { "type": "string", - "title": "Target field", - "description": "The field to write the result to" + "description": "The name of the container within the checkpoint store to store the checkpoints." + }, + "batch_size": { + "type": "integer", + "description": "The maximum number of events to receive in each batch.", + "default": 300 } }, - "additionalProperties": false, - "required": ["connection", "cmd", "args", "language", "field"] + "required": [ + "event_hub_connection_string", + "event_hub_consumer_group_name", + "event_hub_name", + "checkpoint_store_connection_string", + "checkpoint_store_container_name" + ] } } } @@ -734,7 +642,7 @@ "uses": { "description": "Block type", "type": "string", - "const": "redis.read_stream" + "const": "std.write" } }, "required": ["uses"] @@ -742,28 +650,8 @@ "then": { "properties": { "with": { - "title": "redis.read_stream", - "description": "Read from Redis stream", - "type": "object", - "properties": { - "connection": { - "description": "Connection name", - "type": "string" - }, - "stream_name": { - "type": "string", - "title": "Source stream name", - "description": "Source stream name" - }, - "snapshot": { - "type": "boolean", - "title": "Snapshot current entries and quit", - "description": "Snapshot current entries and quit", - "default": false - } - }, - "additionalProperties": false, - "required": ["connection", "stream_name"] + "title": "std.write", + "description": "Write to the standard output" } } } @@ -774,7 +662,7 @@ "uses": { "description": "Block type", "type": "string", - "const": "remove_field" + "const": "std.read" } }, "required": ["uses"] @@ -782,104 +670,47 @@ "then": { "properties": { "with": { - "title": "remove_field", - "description": "Remove fields", - "type": "object", - "oneOf": [ - { - "description": "Remove multiple fields", - "properties": { - "fields": { - "type": "array", - "description": "Fields", - "items": { - "type": "object", - "properties": { - "field": { - "description": "Field", - "type": "string" - } - }, - "additionalProperties": false, - "required": ["field"] - } - } - }, - "required": ["fields"], - "additionalProperties": false, - "examples": [ - { - "fields": [ - { "field": "credit_card" }, - { "field": "name.mname" } - ] - } - ] - }, - { - "description": "Remove one field", - "properties": { - "field": { "description": "Field", "type": "string" } - }, - "additionalProperties": false, - "required": ["field"], - "examples": [{ "field": "credit_card" }] - } - ] + "title": "std.read", + "description": "Read from the standard input" } } } }, { "if": { - "properties": { - "uses": { - "description": "Block type", - "type": "string", - "const": "azure.read_event_hub" - } - }, - "required": ["uses"] - }, - "then": { - "properties": { - "with": { - "title": "azure.read_event_hub", - "description": "Read from Azure Event Hub", - "type": "object", - "properties": { - "event_hub_connection_string": { - "type": "string", - "description": "The connection string for the Azure Event Hub namespace." - }, - "event_hub_consumer_group_name": { - "type": "string", - "description": "The name of the consumer group to read events from." - }, - "event_hub_name": { - "type": "string", - "description": "The name of the Azure Event Hub." - }, - "checkpoint_store_connection_string": { - "type": "string", - "description": "The connection string for the Azure Storage account used as the checkpoint store." - }, - "checkpoint_store_container_name": { - "type": "string", - "description": "The name of the container within the checkpoint store to store the checkpoints." - }, - "batch_size": { - "type": "integer", - "description": "The maximum number of events to receive in each batch.", - "default": 300 + "properties": { + "uses": { + "description": "Block type", + "type": "string", + "const": "jinja_template" + } + }, + "required": ["uses"] + }, + "then": { + "properties": { + "with": { + "title": "jinja_template", + "description": "Apply Jinja template to a field", + "type": "object", + "properties": { + "field": { "description": "Field", "type": "string" }, + "template": { + "description": "Jinja Template", + "type": "string" } }, - "required": [ - "event_hub_connection_string", - "event_hub_consumer_group_name", - "event_hub_name", - "checkpoint_store_connection_string", - "checkpoint_store_container_name" + "additionalProperties": false, + "required": ["field", "template"], + "examples": [ + { + "field": "name.full_name", + "template": "{{ name.fname }} {{ name.lname }}" + }, + { + "field": "name.fname_upper", + "template": "{{ name.fname | upper }}" + } ] } } @@ -976,22 +807,263 @@ "description": "Used for `TYPE2` load_strategy. An SQL expression used to identify which rows are active", "examples": ["is_active='Y'", "deletedAt is null"] }, - "inactive_record_mapping": { - "type": "array", - "title": "Used for `TYPE2` load_strategy. The columns mapping to use to close out an active record", - "description": "A list of columns to use. Use any valid SQL expression for the source. If 'target' is omitted, will default to the name of the source column", - "default": [], + "inactive_record_mapping": { + "type": "array", + "title": "Used for `TYPE2` load_strategy. The columns mapping to use to close out an active record", + "description": "A list of columns to use. Use any valid SQL expression for the source. If 'target' is omitted, will default to the name of the source column", + "default": [], + "examples": [ + [ + { "source": "CURRENT_DATE", "target": "deletedAt" }, + { "source": "'Y'", "target": "is_active" } + ] + ] + } + }, + "required": ["connection", "table"], + "allOf": [ + { "not": { "required": ["opcode_field", "load_strategy"] } } + ] + } + } + } + }, + { + "if": { + "properties": { + "uses": { + "description": "Block type", + "type": "string", + "const": "relational.read" + } + }, + "required": ["uses"] + }, + "then": { + "properties": { + "with": { + "title": "relational.read", + "description": "Read a table from an SQL-compatible data store", + "type": "object", + "additionalProperties": false, + "examples": [ + { + "id": "read_snowflake", + "type": "relational.read", + "properties": { + "connection": "eu_datalake", + "table": "employees", + "schema": "dbo" + } + } + ], + "properties": { + "connection": { + "type": "string", + "title": "The connection to use for loading", + "description": "Logical connection name as defined in the connections.yaml", + "examples": ["europe_db", "target", "eu_dwh"] + }, + "schema": { + "type": "string", + "title": "The table schema of the table", + "description": "If left blank, the default schema of this connection will be used as defined in the connections.yaml", + "examples": ["dbo"] + }, + "table": { + "type": "string", + "title": "The table name", + "description": "Table name", + "examples": ["employees"] + }, + "columns": { + "type": "array", + "title": "Optional subset of columns to load", + "items": { + "type": ["string", "object"], + "title": "name of column" + }, + "examples": [["fname", { "lname": "last_name" }]] + } + }, + "required": ["connection", "table"] + } + } + } + }, + { + "if": { + "properties": { + "uses": { + "description": "Block type", + "type": "string", + "const": "rename_field" + } + }, + "required": ["uses"] + }, + "then": { + "properties": { + "with": { + "title": "rename_field", + "description": "Renames fields. All other fields remain unchanged", + "type": "object", + "oneOf": [ + { + "description": "Rename multiple fields", + "properties": { + "fields": { + "type": "array", + "description": "Fields", + "items": { + "type": "object", + "properties": { + "from_field": { + "description": "From field", + "type": "string" + }, + "to_field": { + "description": "To field", + "type": "string" + } + }, + "additionalProperties": false, + "required": ["from_field", "to_field"], + "examples": [ + { + "fields": [ + { + "from_field": "name.lname", + "to_field": "name.last_name" + }, + { + "from_field": "name.fname", + "to_field": "name.first_name" + } + ] + } + ] + } + } + }, + "required": ["fields"], + "additionalProperties": false + }, + { + "description": "Rename one field", + "properties": { + "from_field": { + "description": "From field", + "type": "string" + }, + "to_field": { + "description": "To field", + "type": "string" + } + }, + "additionalProperties": false, + "required": ["from_field", "to_field"], + "examples": [ + { + "from_field": "name.lname", + "to_field": "name.last_name" + } + ] + } + ] + } + } + } + }, + { + "if": { + "properties": { + "uses": { + "description": "Block type", + "type": "string", + "const": "add_field" + } + }, + "required": ["uses"] + }, + "then": { + "properties": { + "with": { + "title": "add_field", + "description": "Add fields to a record", + "type": "object", + "oneOf": [ + { + "description": "Add multiple fields", + "properties": { + "fields": { + "type": "array", + "description": "Fields", + "items": { + "type": "object", + "properties": { + "field": { + "description": "Field", + "type": "string" + }, + "expression": { + "description": "Expression", + "type": "string" + }, + "language": { + "description": "Language", + "type": "string", + "enum": ["jmespath", "sql"] + } + }, + "additionalProperties": false, + "required": ["field", "expression", "language"] + } + } + }, + "required": ["fields"], + "additionalProperties": false, + "examples": [ + { + "fields": [ + { + "field": "name.full_name", + "language": "jmespath", + "expression": "concat([name.fname, ' ', name.lname])" + }, + { + "field": "name.fname_upper", + "language": "jmespath", + "expression": "upper(name.fname)" + } + ] + } + ] + }, + { + "description": "Add one field", + "properties": { + "field": { "description": "Field", "type": "string" }, + "expression": { + "description": "Expression", + "type": "string" + }, + "language": { + "description": "Language", + "type": "string", + "enum": ["jmespath", "sql"] + } + }, + "additionalProperties": false, + "required": ["field", "expression", "language"], "examples": [ - [ - { "source": "CURRENT_DATE", "target": "deletedAt" }, - { "source": "'Y'", "target": "is_active" } - ] + { + "field": "country", + "language": "sql", + "expression": "country_code || ' - ' || UPPER(country_name)" + } ] } - }, - "required": ["connection", "table"], - "allOf": [ - { "not": { "required": ["opcode_field", "load_strategy"] } } ] } } @@ -1003,7 +1075,7 @@ "uses": { "description": "Block type", "type": "string", - "const": "relational.read" + "const": "sequence" } }, "required": ["uses"] @@ -1011,51 +1083,29 @@ "then": { "properties": { "with": { - "title": "relational.read", - "description": "Read a table from an SQL-compatible data store", + "title": "sequence", + "description": "Add a sequence number field to data", "type": "object", "additionalProperties": false, - "examples": [ - { - "id": "read_snowflake", - "type": "relational.read", - "properties": { - "connection": "eu_datalake", - "table": "employees", - "schema": "dbo" - } - } - ], + "examples": [], + "required": [], "properties": { - "connection": { - "type": "string", - "title": "The connection to use for loading", - "description": "Logical connection name as defined in the connections.yaml", - "examples": ["europe_db", "target", "eu_dwh"] - }, - "schema": { + "field": { "type": "string", - "title": "The table schema of the table", - "description": "If left blank, the default schema of this connection will be used as defined in the connections.yaml", - "examples": ["dbo"] + "title": "Name of new sequence field" }, - "table": { - "type": "string", - "title": "The table name", - "description": "Table name", - "examples": ["employees"] + "start": { + "type": "number", + "title": "Start entry", + "default": 1, + "examples": [] }, - "columns": { - "type": "array", - "title": "Optional subset of columns to load", - "items": { - "type": ["string", "object"], - "title": "name of column" - }, - "examples": [["fname", { "lname": "last_name" }]] + "increment": { + "type": "number", + "title": "Increment between sequences", + "examples": [] } - }, - "required": ["connection", "table"] + } } } } @@ -1066,7 +1116,7 @@ "uses": { "description": "Block type", "type": "string", - "const": "map" + "const": "files.read_csv" } }, "required": ["uses"] @@ -1074,38 +1124,63 @@ "then": { "properties": { "with": { - "title": "map", - "description": "Maps a record into a new output based on expressions", + "title": "files.read_csv", + "description": "Read data from CSV", "type": "object", "properties": { - "expression": { - "description": "Expression", - "type": ["object", "string"] + "file": { + "description": "Filename. Can contain a regexp or glob expression", + "type": "string" }, - "language": { - "description": "Language", + "encoding": { + "description": "Encoding to use for reading the file", "type": "string", - "enum": ["jmespath", "sql"] + "default": "utf-8" + }, + "fields": { + "type": "array", + "title": "List of columns to use", + "description": "List of columns to use for extract", + "default": null, + "examples": [["fname", "lname"]], + "minLength": 1, + "additionalItems": true, + "items": { + "type": "string", + "description": "field name", + "examples": ["fname"] + } + }, + "skip": { + "description": "Number of lines to skip", + "type": "number", + "minimum": 0, + "default": 0 + }, + "delimiter": { + "description": "Delimiter to use for splitting the csv records", + "type": "string", + "minLength": 1, + "maxLength": 1, + "default": "," + }, + "batch_size": { + "description": "Number of records to read per batch", + "type": "number", + "minimum": 1, + "default": 1000 + }, + "quotechar": { + "description": "A one-character string used to quote fields containing special characters, such as the delimiter or quotechar, or which contain new-line characters. It defaults to '", + "type": "string", + "minLength": 1, + "maxLength": 1, + "default": "\"" } }, "additionalProperties": false, - "required": ["expression", "language"], - "examples": [ - { - "expression": { - "first_name": "first_name", - "last_name": "last_name", - "greeting": "'Hello ' || CASE WHEN gender = 'F' THEN 'Ms.' WHEN gender = 'M' THEN 'Mr.' ELSE 'N/A' END || ' ' || full_name", - "country": "country", - "full_name": "full_name" - }, - "language": "sql" - }, - { - "expression": "{\"CustomerId\": \"customer_id\", \"FirstName\": \"first_name\", \"LastName\": \"last_name\", \"Company\": \"company\", \"Location\": {\"Street\": \"address\", \"City\": \"city\", \"State\": \"state\", \"Country\": \"country\", \"PostalCode\": \"postal_code\"}, \"Phone\": \"phone\", \"Fax\": \"fax\", \"Email\": \"email\"}", - "language": "jmespath" - } - ] + "required": ["file"], + "examples": [{ "file": "archive.csv", "delimiter": ";" }] } } } @@ -1255,81 +1330,6 @@ } } } - }, - { - "if": { - "properties": { - "uses": { - "description": "Block type", - "type": "string", - "const": "files.read_csv" - } - }, - "required": ["uses"] - }, - "then": { - "properties": { - "with": { - "title": "files.read_csv", - "description": "Read data from CSV", - "type": "object", - "properties": { - "file": { - "description": "Filename. Can contain a regexp or glob expression", - "type": "string" - }, - "encoding": { - "description": "Encoding to use for reading the file", - "type": "string", - "default": "utf-8" - }, - "fields": { - "type": "array", - "title": "List of columns to use", - "description": "List of columns to use for extract", - "default": null, - "examples": [["fname", "lname"]], - "minLength": 1, - "additionalItems": true, - "items": { - "type": "string", - "description": "field name", - "examples": ["fname"] - } - }, - "skip": { - "description": "Number of lines to skip", - "type": "number", - "minimum": 0, - "default": 0 - }, - "delimiter": { - "description": "Delimiter to use for splitting the csv records", - "type": "string", - "minLength": 1, - "maxLength": 1, - "default": "," - }, - "batch_size": { - "description": "Number of records to read per batch", - "type": "number", - "minimum": 1, - "default": 1000 - }, - "quotechar": { - "description": "A one-character string used to quote fields containing special characters, such as the delimiter or quotechar, or which contain new-line characters. It defaults to '", - "type": "string", - "minLength": 1, - "maxLength": 1, - "default": "\"" - } - }, - "additionalProperties": false, - "required": ["file"], - "examples": [{ "file": "archive.csv", "delimiter": ";" }] - } - } - } } ] }