From 9249c43fb5e68e1889b479df5bb70cb2e93f8d76 Mon Sep 17 00:00:00 2001 From: Marvin Buss <34542414+marvinbuss@users.noreply.github.com> Date: Mon, 27 May 2024 15:44:26 +0200 Subject: [PATCH 1/6] Updating pipeline: TestOttRawToCurated --- code/datafactory/pipeline/TestOttRawToCurated.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/code/datafactory/pipeline/TestOttRawToCurated.json b/code/datafactory/pipeline/TestOttRawToCurated.json index f1fc247..bd6d67c 100644 --- a/code/datafactory/pipeline/TestOttRawToCurated.json +++ b/code/datafactory/pipeline/TestOttRawToCurated.json @@ -113,7 +113,7 @@ }, "sourceFolderPath": { "type": "string", - "defaultValue": "ott" + "defaultValue": "ott/751cb63e-ce65-4f73-9894-6b1bdb8b7e1d" }, "sourceFileName": { "type": "string", From 5c12a06103adf23df084abed0ac960b72ee343f9 Mon Sep 17 00:00:00 2001 From: Marvin Buss <34542414+marvinbuss@users.noreply.github.com> Date: Mon, 27 May 2024 16:01:56 +0200 Subject: [PATCH 2/6] Updating dataflow: IptvRawToCurated --- code/datafactory/dataflow/IptvRawToCurated.json | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/code/datafactory/dataflow/IptvRawToCurated.json b/code/datafactory/dataflow/IptvRawToCurated.json index 2c3294b..9e1e31d 100644 --- a/code/datafactory/dataflow/IptvRawToCurated.json +++ b/code/datafactory/dataflow/IptvRawToCurated.json @@ -50,6 +50,7 @@ "source(output(", " app_version as string,", " country as string,", + " end_time as timestamp 'yyyy-MM-dd\\'T\\'HH:mm:ss.SSS\\'Z\\'',", " happiness_score as float,", " hw_model as string,", " service as string,", @@ -91,7 +92,8 @@ " Topology4 = topology_4,", " Topology5 = topology_5,", " TvModel = tv_model,", - " UserId = user_id", + " UserId = user_id,", + " EndTime = end_time", " ),", " skipDuplicateMapInputs: true,", " skipDuplicateMapOutputs: true) ~> RenameColumns", From 09e6a90571635e10ce5562c3b82331a2ba602b5e Mon Sep 17 00:00:00 2001 From: Marvin Buss <34542414+marvinbuss@users.noreply.github.com> Date: Mon, 27 May 2024 16:05:09 +0200 Subject: [PATCH 3/6] Updating dataflow: OttCuratedToKusto --- code/datafactory/dataflow/OttCuratedToKusto.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/code/datafactory/dataflow/OttCuratedToKusto.json b/code/datafactory/dataflow/OttCuratedToKusto.json index 7908ff9..b140675 100644 --- a/code/datafactory/dataflow/OttCuratedToKusto.json +++ b/code/datafactory/dataflow/OttCuratedToKusto.json @@ -52,7 +52,8 @@ " CommercilizationType as string,", " DeviceVendor as string,", " HappinessScore as float,", - " UserId as string", + " UserId as long,", + " EndTime as timestamp", " ),", " allowSchemaDrift: true,", " validateSchema: false,", From 5f0a3a2550c95d8dbdec1bc790bd5c0e221a4835 Mon Sep 17 00:00:00 2001 From: Marvin Buss <34542414+marvinbuss@users.noreply.github.com> Date: Mon, 27 May 2024 16:05:45 +0200 Subject: [PATCH 4/6] Updating pipeline: TestIptvRawToCurated --- code/datafactory/pipeline/TestIptvRawToCurated.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/code/datafactory/pipeline/TestIptvRawToCurated.json b/code/datafactory/pipeline/TestIptvRawToCurated.json index 146db81..509a95a 100644 --- a/code/datafactory/pipeline/TestIptvRawToCurated.json +++ b/code/datafactory/pipeline/TestIptvRawToCurated.json @@ -113,7 +113,7 @@ }, "sourceFolderPath": { "type": "string", - "defaultValue": "iptv" + "defaultValue": "iptv/0156490b-efbe-4a27-af08-a237669f91cc" }, "sourceFileName": { "type": "string", From 5ab7a39c05b79ef8e4ac4df016f8e3dc554a2bb8 Mon Sep 17 00:00:00 2001 From: Marvin Buss <34542414+marvinbuss@users.noreply.github.com> Date: Mon, 27 May 2024 16:11:03 +0200 Subject: [PATCH 5/6] Updating dataflow: IptvCuratedToKusto From 2b82b7ab14d1b011bf574776a22f7650cb40589d Mon Sep 17 00:00:00 2001 From: Marvin Buss <34542414+marvinbuss@users.noreply.github.com> Date: Mon, 27 May 2024 16:12:20 +0200 Subject: [PATCH 6/6] Updating dataflow: IptvCuratedToKusto --- code/datafactory/dataflow/IptvCuratedToKusto.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/code/datafactory/dataflow/IptvCuratedToKusto.json b/code/datafactory/dataflow/IptvCuratedToKusto.json index eb32250..ea9f851 100644 --- a/code/datafactory/dataflow/IptvCuratedToKusto.json +++ b/code/datafactory/dataflow/IptvCuratedToKusto.json @@ -54,7 +54,8 @@ " Topology4 as string,", " Topology5 as string,", " TvModel as string,", - " UserId as long", + " UserId as long,", + " EndTime as timestamp", " ),", " allowSchemaDrift: true,", " validateSchema: false,",