Merge pull request #21 from PerfectThymeTech/marvinbuss/add_end_time

Add End Time to Data Model
PerfectThymeTech · May 27, 2024 · d1a4124 · d1a4124
2 parents 1413d03 + 84fce7a
commit d1a4124
Show file tree

Hide file tree

Showing 7 changed files with 16 additions and 8 deletions.
diff --git a/code/datafactory/dataflow/IptvCuratedToKusto.json b/code/datafactory/dataflow/IptvCuratedToKusto.json
@@ -54,7 +54,8 @@
 				"          Topology4 as string,",
 				"          Topology5 as string,",
 				"          TvModel as string,",
-				"          UserId as long",
+				"          UserId as long,",
+				"          EndTime as timestamp",
 				"     ),",
 				"     allowSchemaDrift: true,",
 				"     validateSchema: false,",

diff --git a/code/datafactory/dataflow/IptvRawToCurated.json b/code/datafactory/dataflow/IptvRawToCurated.json
@@ -50,6 +50,7 @@
 				"source(output(",
 				"          app_version as string,",
 				"          country as string,",
+				"          end_time as timestamp 'yyyy-MM-dd\\'T\\'HH:mm:ss.SSS\\'Z\\'',",
 				"          happiness_score as float,",
 				"          hw_model as string,",
 				"          service as string,",
@@ -91,7 +92,8 @@
 				"          Topology4 = topology_4,",
 				"          Topology5 = topology_5,",
 				"          TvModel = tv_model,",
-				"          UserId = user_id",
+				"          UserId = user_id,",
+				"          EndTime = end_time",
 				"     ),",
 				"     skipDuplicateMapInputs: true,",
 				"     skipDuplicateMapOutputs: true) ~> RenameColumns",

diff --git a/code/datafactory/dataflow/OttCuratedToKusto.json b/code/datafactory/dataflow/OttCuratedToKusto.json
@@ -52,7 +52,8 @@
 				"          CommercilizationType as string,",
 				"          DeviceVendor as string,",
 				"          HappinessScore as float,",
-				"          UserId as string",
+				"          UserId as long,",
+				"          EndTime as timestamp",
 				"     ),",
 				"     allowSchemaDrift: true,",
 				"     validateSchema: false,",

diff --git a/code/datafactory/dataflow/OttRawToCurated.json b/code/datafactory/dataflow/OttRawToCurated.json
@@ -60,7 +60,8 @@
 				"          {COMMERCIALIZATION TYPE} as string,",
 				"          {Device Vendor} as string,",
 				"          {Happiness Score} as float,",
-				"          {User ID} as string",
+				"          {User ID} as long,",
+				"          {End Time} as timestamp",
 				"     ),",
 				"     useSchema: false,",
 				"     allowSchemaDrift: true,",
@@ -87,7 +88,8 @@
 				"          CommercilizationType = {COMMERCIALIZATION TYPE},",
 				"          DeviceVendor = {Device Vendor},",
 				"          HappinessScore = {Happiness Score},",
-				"          UserId = {User ID}",
+				"          UserId = {User ID},",
+				"          EndTime = {End Time}",
 				"     ),",
 				"     skipDuplicateMapInputs: true,",
 				"     skipDuplicateMapOutputs: true) ~> RenameColumns",

diff --git a/code/datafactory/pipeline/TestIptvRawToCurated.json b/code/datafactory/pipeline/TestIptvRawToCurated.json
@@ -113,7 +113,7 @@
 			},
 			"sourceFolderPath": {
 				"type": "string",
-				"defaultValue": "iptv"
+				"defaultValue": "iptv/0156490b-efbe-4a27-af08-a237669f91cc"
 			},
 			"sourceFileName": {
 				"type": "string",

diff --git a/code/datafactory/pipeline/TestOttRawToCurated.json b/code/datafactory/pipeline/TestOttRawToCurated.json
@@ -113,7 +113,7 @@
 			},
 			"sourceFolderPath": {
 				"type": "string",
-				"defaultValue": "ott"
+				"defaultValue": "ott/751cb63e-ce65-4f73-9894-6b1bdb8b7e1d"
 			},
 			"sourceFileName": {
 				"type": "string",

diff --git a/code/datamodel/operationaldb.kql b/code/datamodel/operationaldb.kql
@@ -13,11 +13,13 @@
         CommercilizationType: string,
         DeviceVendor: string,
         HappinessScore: real,
-        UserId: string
+        UserId: string,
+        EndTime: datetime
     ),
     iptv_raw(
         AppVersion: string,
         Country: string,
+        EndTime: datetime,
         HappinessScore: real,
         HwModel: string,
         Service: string,