From 0845db7e224e0e4ca706713e8e5d63313f386985 Mon Sep 17 00:00:00 2001 From: Marvin Buss <34542414+marvinbuss@users.noreply.github.com> Date: Fri, 31 May 2024 17:27:01 +0200 Subject: [PATCH 1/2] ARM template and parameters deployed on 5-31-2024 17:26:56, based on the collaboration branch's commit ID: d84498cd085655ff9a27435d72980b2e40123d58 --- sc4-dev-df001/ARMTemplateForFactory.json | 8 +++++++- sc4-dev-df001/ARMTemplateParametersForFactory.json | 14 +++++++------- sc4-dev-df001/linkedTemplates/ArmTemplate_0.json | 8 +++++++- 3 files changed, 21 insertions(+), 9 deletions(-) diff --git a/sc4-dev-df001/ARMTemplateForFactory.json b/sc4-dev-df001/ARMTemplateForFactory.json index 689b1ac..3f97097 100644 --- a/sc4-dev-df001/ARMTemplateForFactory.json +++ b/sc4-dev-df001/ARMTemplateForFactory.json @@ -4193,6 +4193,10 @@ { "name": "FilterEmptyHappinessScore", "description": "Filter Empty Happiness Score Rows" + }, + { + "name": "HarmonizeColumn", + "description": "Harmonize columns: Connection, DeviceVendor" } ], "scriptLines": [ @@ -4252,7 +4256,9 @@ " skipDuplicateMapInputs: true,", " skipDuplicateMapOutputs: true) ~> RenameColumns", "RenameColumns filter(!isNull(HappinessScore)) ~> FilterEmptyHappinessScore", - "FilterEmptyHappinessScore sink(allowSchemaDrift: true,", + "FilterEmptyHappinessScore derive(Connection = lower(Connection),", + " DeviceVendor = lower(DeviceVendor)) ~> HarmonizeColumn", + "HarmonizeColumn sink(allowSchemaDrift: true,", " validateSchema: false,", " format: 'delta',", " fileSystem: ($sinkFileSystem),", diff --git a/sc4-dev-df001/ARMTemplateParametersForFactory.json b/sc4-dev-df001/ARMTemplateParametersForFactory.json index a6d774a..8a2c5f6 100644 --- a/sc4-dev-df001/ARMTemplateParametersForFactory.json +++ b/sc4-dev-df001/ARMTemplateParametersForFactory.json @@ -3,19 +3,19 @@ "contentVersion": "1.0.0.0", "parameters": { "factoryName": { - "value": "${data_factory_name}" + "value": "sc4-dev-df001" }, "AzureDataExplorer_properties_typeProperties_endpoint": { - "value": "${kusto_cluster_uri}" + "value": "https://sc4-dev-kusto001.germanywestcentral.kusto.windows.net" }, "AzureDataExplorer_properties_typeProperties_database": { "value": "@linkedService().databaseName" }, "AzureDataLakeStorage_properties_typeProperties_url": { - "value": "${datalake_primary_dfs_endpoint}" + "value": "https://sc4devstg001.dfs.core.windows.net/" }, "AzureKeyVault_properties_typeProperties_baseUrl": { - "value": "${key_vault_uri}" + "value": "https://sc4-dev-vault001.vault.azure.net/" }, "IptvUpload_properties_IptvMainPipeline_parameters_uploadSourceFolderPath": { "value": "@triggerBody().folderPath" @@ -24,7 +24,7 @@ "value": "@triggerBody().fileName" }, "IptvUpload_properties_typeProperties_scope": { - "value": "${datalake_id}" + "value": "/subscriptions/be25820a-df86-4794-9e95-6a45cd5c0941/resourceGroups/sc4-dev-stg-rg/providers/Microsoft.Storage/storageAccounts/sc4devstg001" }, "OttUpload_properties_OttMainPipeline_parameters_uploadSourceFolderPath": { "value": "@triggerBody().folderPath" @@ -33,7 +33,7 @@ "value": "@triggerBody().fileName" }, "OttUpload_properties_typeProperties_scope": { - "value": "${datalake_id}" + "value": "/subscriptions/be25820a-df86-4794-9e95-6a45cd5c0941/resourceGroups/sc4-dev-stg-rg/providers/Microsoft.Storage/storageAccounts/sc4devstg001" }, "TestUpload_properties_TestParseBlobTrigger_parameters_uploadSourceFolderPath": { "value": "@triggerBody().folderPath" @@ -42,7 +42,7 @@ "value": "@triggerBody().fileName" }, "TestUpload_properties_typeProperties_scope": { - "value": "${datalake_id}" + "value": "/subscriptions/be25820a-df86-4794-9e95-6a45cd5c0941/resourceGroups/sc4-dev-stg-rg/providers/Microsoft.Storage/storageAccounts/sc4devstg001" } } } \ No newline at end of file diff --git a/sc4-dev-df001/linkedTemplates/ArmTemplate_0.json b/sc4-dev-df001/linkedTemplates/ArmTemplate_0.json index 550ed87..3e224c5 100644 --- a/sc4-dev-df001/linkedTemplates/ArmTemplate_0.json +++ b/sc4-dev-df001/linkedTemplates/ArmTemplate_0.json @@ -1043,6 +1043,10 @@ { "name": "FilterEmptyHappinessScore", "description": "Filter Empty Happiness Score Rows" + }, + { + "name": "HarmonizeColumn", + "description": "Harmonize columns: Connection, DeviceVendor" } ], "scriptLines": [ @@ -1102,7 +1106,9 @@ " skipDuplicateMapInputs: true,", " skipDuplicateMapOutputs: true) ~> RenameColumns", "RenameColumns filter(!isNull(HappinessScore)) ~> FilterEmptyHappinessScore", - "FilterEmptyHappinessScore sink(allowSchemaDrift: true,", + "FilterEmptyHappinessScore derive(Connection = lower(Connection),", + " DeviceVendor = lower(DeviceVendor)) ~> HarmonizeColumn", + "HarmonizeColumn sink(allowSchemaDrift: true,", " validateSchema: false,", " format: 'delta',", " fileSystem: ($sinkFileSystem),", From 4467460ff0ced9891a1357970a7251b0a7afdc20 Mon Sep 17 00:00:00 2001 From: Marvin Buss Date: Fri, 31 May 2024 18:20:20 +0200 Subject: [PATCH 2/2] Revert template file --- sc4-dev-df001/ARMTemplateParametersForFactory.json | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/sc4-dev-df001/ARMTemplateParametersForFactory.json b/sc4-dev-df001/ARMTemplateParametersForFactory.json index 8a2c5f6..a6d774a 100644 --- a/sc4-dev-df001/ARMTemplateParametersForFactory.json +++ b/sc4-dev-df001/ARMTemplateParametersForFactory.json @@ -3,19 +3,19 @@ "contentVersion": "1.0.0.0", "parameters": { "factoryName": { - "value": "sc4-dev-df001" + "value": "${data_factory_name}" }, "AzureDataExplorer_properties_typeProperties_endpoint": { - "value": "https://sc4-dev-kusto001.germanywestcentral.kusto.windows.net" + "value": "${kusto_cluster_uri}" }, "AzureDataExplorer_properties_typeProperties_database": { "value": "@linkedService().databaseName" }, "AzureDataLakeStorage_properties_typeProperties_url": { - "value": "https://sc4devstg001.dfs.core.windows.net/" + "value": "${datalake_primary_dfs_endpoint}" }, "AzureKeyVault_properties_typeProperties_baseUrl": { - "value": "https://sc4-dev-vault001.vault.azure.net/" + "value": "${key_vault_uri}" }, "IptvUpload_properties_IptvMainPipeline_parameters_uploadSourceFolderPath": { "value": "@triggerBody().folderPath" @@ -24,7 +24,7 @@ "value": "@triggerBody().fileName" }, "IptvUpload_properties_typeProperties_scope": { - "value": "/subscriptions/be25820a-df86-4794-9e95-6a45cd5c0941/resourceGroups/sc4-dev-stg-rg/providers/Microsoft.Storage/storageAccounts/sc4devstg001" + "value": "${datalake_id}" }, "OttUpload_properties_OttMainPipeline_parameters_uploadSourceFolderPath": { "value": "@triggerBody().folderPath" @@ -33,7 +33,7 @@ "value": "@triggerBody().fileName" }, "OttUpload_properties_typeProperties_scope": { - "value": "/subscriptions/be25820a-df86-4794-9e95-6a45cd5c0941/resourceGroups/sc4-dev-stg-rg/providers/Microsoft.Storage/storageAccounts/sc4devstg001" + "value": "${datalake_id}" }, "TestUpload_properties_TestParseBlobTrigger_parameters_uploadSourceFolderPath": { "value": "@triggerBody().folderPath" @@ -42,7 +42,7 @@ "value": "@triggerBody().fileName" }, "TestUpload_properties_typeProperties_scope": { - "value": "/subscriptions/be25820a-df86-4794-9e95-6a45cd5c0941/resourceGroups/sc4-dev-stg-rg/providers/Microsoft.Storage/storageAccounts/sc4devstg001" + "value": "${datalake_id}" } } } \ No newline at end of file