Fix Duplicate conflicting deployments with DeployAllMultipleTemplateP…

…arameterFiles (#887)

* Update

* Update

* Update

src/functions/Invoke-AzOpsPush.ps1

@@ -158,7 +158,7 @@
             #region Case: Parameters File
             if (($fileItem.Name.EndsWith('.parameters.json')) -or ($fileItem.Name.EndsWith('.bicepparam'))) {
                 $result.TemplateParameterFilePath = $fileItem.FullName
-                $deploymentName = $fileItem.Name -replace (Get-PSFConfigValue -FullName 'AzOps.Core.TemplateParameterFileSuffix'), '' -replace ' ', '_' -replace '\.bicepparam', ''
+                $deploymentName = $fileItem.Name -replace "\.parameters\$(Get-PSFConfigValue -FullName 'AzOps.Core.TemplateParameterFileSuffix')$", '' -replace ' ', '_' -replace '\.bicepparam$', ''
                 if ($deploymentName.Length -gt 53) { $deploymentName = $deploymentName.SubString(0, 53) }
                 $result.DeploymentName = 'AzOps-{0}-{1}' -f $deploymentName, $deploymentRegionId
 
@@ -167,12 +167,12 @@
                     { $_.EndsWith('.parameters.json') } {
                         if ((Get-PSFConfigValue -FullName 'AzOps.Core.AllowMultipleTemplateParameterFiles') -eq $true -and $fileItem.FullName.Split('.')[-3] -match $(Get-PSFConfigValue -FullName 'AzOps.Core.MultipleTemplateParameterFileSuffix').Replace('.','')) {
                             Write-AzOpsMessage -LogLevel Debug -LogString 'Invoke-AzOpsPush.Resolve.MultipleTemplateParameterFile' -LogStringValues $FilePath
-                            $templatePath = $fileItem.FullName -replace "\.$($fileItem.FullName.Split('.')[-3])\.parameters.json$", '.json'
-                            $bicepTemplatePath = $fileItem.FullName -replace "\.$($fileItem.FullName.Split('.')[-3])\.parameters.json$", '.bicep'
+                            $templatePath = $fileItem.FullName -replace "\.$($fileItem.FullName.Split('.')[-3])\.parameters\.json$", '.json'
+                            $bicepTemplatePath = $fileItem.FullName -replace "\.$($fileItem.FullName.Split('.')[-3])\.parameters\.json$", '.bicep'
                         }
                         else {
-                            $templatePath = $fileItem.FullName -replace '\.parameters.json$', (Get-PSFConfigValue -FullName 'AzOps.Core.TemplateParameterFileSuffix')
-                            $bicepTemplatePath = $fileItem.FullName -replace '\.parameters.json$', '.bicep'
+                            $templatePath = $fileItem.FullName -replace '\.parameters\.json$', (Get-PSFConfigValue -FullName 'AzOps.Core.TemplateParameterFileSuffix')
+                            $bicepTemplatePath = $fileItem.FullName -replace '\.parameters\.json$', '.bicep'
                         }
                         if (Test-Path $templatePath) {
                             if ($CompareDeploymentToDeletion) {
@@ -299,17 +299,21 @@
                 if ($paramFileList) {
                     $multiResult = @()
                     foreach ($paramFile in $paramFileList) {
-                        if ($CompareDeploymentToDeletion) {
-                            # Avoid adding files destined for deletion to a deployment list
-                            if ($paramFile.VersionInfo.FileName -in $deleteSet -or $paramFile.VersionInfo.FileName -in ($deleteSet | Resolve-Path).Path) {
-                                Write-AzOpsMessage -LogLevel Debug -LogString 'Invoke-AzOpsPush.Resolve.DeployDeletionOverlap' -LogStringValues $paramFile.VersionInfo.FileName
-                                continue
+                        # Check if the parameter file's name matches the expected pattern
+                        $escapedBaseName = $fileItem.BaseName -replace '\.', '\.'
+                        if ($paramFile.BaseName -match "^$escapedBaseName(\$(Get-PSFConfigValue -FullName 'AzOps.Core.MultipleTemplateParameterFileSuffix'))") {
+                            if ($CompareDeploymentToDeletion) {
+                                # Avoid adding files destined for deletion to a deployment list
+                                if ($paramFile.VersionInfo.FileName -in $deleteSet -or $paramFile.VersionInfo.FileName -in ($deleteSet | Resolve-Path).Path) {
+                                    Write-AzOpsMessage -LogLevel Debug -LogString 'Invoke-AzOpsPush.Resolve.DeployDeletionOverlap' -LogStringValues $paramFile.VersionInfo.FileName
+                                    continue
+                                }
+                            }
+                            # Process parameter files for template equivalent
+                            if (($fileItem.FullName.Split('.')[-2] -eq $paramFile.FullName.Split('.')[-3]) -or ($fileItem.FullName.Split('.')[-2] -eq $paramFile.FullName.Split('.')[-4])) {
+                                Write-AzOpsMessage -LogLevel Debug -LogString 'Invoke-AzOpsPush.Resolve.MultipleTemplateParameterFile' -LogStringValues $paramFile.FullName
+                                $multiResult += Resolve-ArmFileAssociation -ScopeObject $scopeObject -FilePath $paramFile -AzOpsMainTemplate $AzOpsMainTemplate -ConvertedTemplate $ConvertedTemplate -ConvertedParameter $ConvertedParameter -CompareDeploymentToDeletion:$CompareDeploymentToDeletion
                             }
-                        }
-                        # Process possible parameter files for template equivalent
-                        if (($fileItem.FullName.Split('.')[-2] -eq $paramFile.FullName.Split('.')[-3]) -or ($fileItem.FullName.Split('.')[-2] -eq $paramFile.FullName.Split('.')[-4])) {
-                            Write-AzOpsMessage -LogLevel Debug -LogString 'Invoke-AzOpsPush.Resolve.MultipleTemplateParameterFile' -LogStringValues $paramFile.FullName
-                            $multiResult += Resolve-ArmFileAssociation -ScopeObject $scopeObject -FilePath $paramFile -AzOpsMainTemplate $AzOpsMainTemplate -ConvertedTemplate $ConvertedTemplate -ConvertedParameter $ConvertedParameter -CompareDeploymentToDeletion:$CompareDeploymentToDeletion
                         }
                     }
                     if ($multiResult) {
@@ -318,20 +322,16 @@
                     }
                     else {
                         Write-AzOpsMessage -LogLevel Debug -LogString 'Invoke-AzOpsPush.Resolve.ParameterNotFound' -LogStringValues $FilePath, $parameterPath
+                        if (-not (Test-TemplateDefaultParameter -FilePath $FilePath)) {
+                            continue
+                        }
                     }
-
                 }
             }
             else {
                 Write-AzOpsMessage -LogLevel Debug -LogString 'Invoke-AzOpsPush.Resolve.ParameterNotFound' -LogStringValues $FilePath, $parameterPath
                 if ((Get-PSFConfigValue -FullName 'AzOps.Core.AllowMultipleTemplateParameterFiles') -eq $true) {
-                    # Check for template parameters without defaultValue
-                    $defaultValueContent = Get-Content $FilePath
-                    $missingDefaultParam = $defaultValueContent | jq '.parameters | with_entries(select(.value.defaultValue == null))' | ConvertFrom-Json -AsHashtable
-                    if ($missingDefaultParam.Count -ge 1) {
-                        # Skip template deployment when template parameters without defaultValue are found and no parameter file identified
-                        $missingString = foreach ($item in $missingDefaultParam.Keys.GetEnumerator()) {"$item,"}
-                        Write-AzOpsMessage -LogLevel Debug -LogString 'Invoke-AzOpsPush.Resolve.NotFoundParamFileDefaultValue' -LogStringValues $FilePath, ($missingString | Out-String -NoNewline)
+                    if (-not (Test-TemplateDefaultParameter -FilePath $FilePath)) {
                         continue
                     }
                 }
@@ -344,6 +344,27 @@
             $result
             #endregion Case: Template File
         }
+        function Test-TemplateDefaultParameter {
+            param(
+                [string]$FilePath
+            )
+
+            # Read the template file
+            $defaultValueContent = Get-Content $FilePath
+            # Check for parameters without a default value using jq
+            $missingDefaultParam = $defaultValueContent | jq '.parameters | with_entries(select(.value.defaultValue == null))' | ConvertFrom-Json -AsHashtable
+            if ($missingDefaultParam.Count -ge 1) {
+                # Skip template deployment when template parameters without defaultValue are found and no parameter file identified
+                $missingString = foreach ($item in $missingDefaultParam.Keys.GetEnumerator()) {"$item,"}
+                # Log a debug message with the missing parameters
+                Write-AzOpsMessage -LogLevel Debug -LogString 'Invoke-AzOpsPush.Resolve.NotFoundParamFileDefaultValue' -LogStringValues $FilePath, ($missingString | Out-String -NoNewline)
+                # Missing default value were found
+                return $false
+            } else {
+                # Default values found
+                return $true
+            }
+        }
         #endregion Utility Functions
 
         $WhatIfPreferenceState = $WhatIfPreference

src/tests/integration/Repository.Tests.ps1

@@ -139,7 +139,7 @@ Describe "Repository" {
             $script:resourceGroupParallelDeploy = (Get-AzResourceGroup | Where-Object ResourceGroupName -eq "ParallelDeploy-azopsrg")
             $script:roleAssignments = (Get-AzRoleAssignment -ObjectId "023e7c1c-1fa4-4818-bb78-0a9c5e8b0217" | Where-Object { $_.Scope -eq "/subscriptions/$script:subscriptionId" -and $_.RoleDefinitionId -eq "acdd72a7-3385-48ef-bd42-f606fba81ae7" })
             $script:policyExemptions = Get-AzPolicyExemption -Name "PolicyExemptionTest" -Scope "/subscriptions/$script:subscriptionId"
-            $script:routeTable = (Get-AzResource -Name "RouteTable" -ResourceGroupName $($script:resourceGroup).ResourceGroupName)
+            $script:routeTable = (Get-AzResource -Name "RouteTable" -ResourceGroupName $script:resourceGroup.ResourceGroupName)
             $script:policyAssignmentsDeletion = Get-AzPolicyAssignment -Name "TestPolicyAssignmentDeletion" -Scope "/subscriptions/$script:subscriptionId/resourceGroups/$($script:resourceGroupCustomDeletion.ResourceGroupName)"
             $script:ruleCollectionGroups = (Get-AzResource -ExpandProperties -Name "TestPolicy" -ResourceGroupName $($script:resourceGroup).ResourceGroupName).Properties.ruleCollectionGroups.id.split("/")[-1]
             $script:logAnalyticsWorkspace = (Get-AzResource -Name "thisisalongloganalyticsworkspacename123456789011121314151617181" -ResourceGroupName $($script:resourceGroup).ResourceGroupName)
@@ -1162,7 +1162,7 @@ Describe "Repository" {
             )
             {Invoke-AzOpsPush -ChangeSet $changeSet} | Should -Not -Throw
             Start-Sleep -Seconds 5
-            $script:bicepMultiParamPathDeployment = Get-AzResource -ResourceGroupName $($script:resourceGroup).ResourceGroupName -ResourceType 'Microsoft.Network/routeTables'  | Where-Object {$_.name -like "rtmultibasex*"}
+            $script:bicepMultiParamPathDeployment = Get-AzResource -ResourceGroupName $script:resourceGroup.ResourceGroupName -ResourceType 'Microsoft.Network/routeTables'  | Where-Object {$_.name -like "rtmultibasex*"}
             $script:bicepMultiParamPathDeployment.Count | Should -Be 2
         }
         #endregion
@@ -1178,7 +1178,7 @@ Describe "Repository" {
             )
             {Invoke-AzOpsPush -ChangeSet $changeSet} | Should -Not -Throw
             Start-Sleep -Seconds 5
-            $script:bicepRepeatSuffixPathDeployment = Get-AzResource -ResourceGroupName $($script:resourceGroup).ResourceGroupName -ResourceType 'Microsoft.Network/routeTables'  | Where-Object {$_.name -like "rtsuffix*"}
+            $script:bicepRepeatSuffixPathDeployment = Get-AzResource -ResourceGroupName $script:resourceGroup.ResourceGroupName -ResourceType 'Microsoft.Network/routeTables'  | Where-Object {$_.name -like "rtsuffix*"}
             $script:bicepRepeatSuffixPathDeployment.Count | Should -Be 2
             Set-PSFConfig -FullName AzOps.Core.MultipleTemplateParameterFileSuffix -Value ".x"
         }
@@ -1214,11 +1214,31 @@ Describe "Repository" {
             )
             {Invoke-AzOpsPush -ChangeSet $changeSet} | Should -Not -Throw
             Start-Sleep -Seconds 5
-            $script:deployAllRtParamPathDeployment = Get-AzResource -ResourceGroupName $($script:resourceGroup).ResourceGroupName -ResourceType 'Microsoft.Network/routeTables'  | Where-Object {$_.name -like "deployallrtbasex*"}
+            $script:deployAllRtParamPathDeployment = Get-AzResource -ResourceGroupName $script:resourceGroup.ResourceGroupName -ResourceType 'Microsoft.Network/routeTables'  | Where-Object {$_.name -like "deployallrtbasex*"}
             $script:deployAllRtParamPathDeployment.Count | Should -Be 2
         }
         #endregion
 
+        #region Bicep template with change, AzOps set to resolve corresponding parameter files and create multiple deployments [3] and avoid [1] decoy
+        It "Deploy Bicep template with change, AzOps set to resolve corresponding parameter files and create multiple deployments [3] and avoid [1] decoy" {
+            Set-PSFConfig -FullName AzOps.Core.AllowMultipleTemplateParameterFiles -Value $true
+            Set-PSFConfig -FullName AzOps.Core.DeployAllMultipleTemplateParameterFiles -Value $true
+            $script:deployAllRtFilesPath = Get-ChildItem -Path "$($global:testRoot)/templates/deployallrt.westeurope*" | Copy-Item -Destination $script:resourceGroupDirectory -PassThru -Force
+            $script:deployAllRt2FilesPath = Get-ChildItem -Path "$($global:testRoot)/templates/deployallrt2.westeurope.bicep" | Copy-Item -Destination $script:resourceGroupDirectory -PassThru -Force
+            $script:decoyRtFilesPath = Get-ChildItem -Path "$($global:testRoot)/templates/decoy.westeurope*" | Copy-Item -Destination $script:resourceGroupDirectory -PassThru -Force
+            $changeSet = @(
+                "A`t$($script:deployAllRtFilesPath.FullName[0])",
+                "A`t$($script:deployAllRt2FilesPath.FullName)"
+            )
+            $testFiles = Invoke-AzOpsPush -ChangeSet $changeSet
+            $? | Should -Be $true
+            $testFiles.Count | Should -Be 3
+            Start-Sleep -Seconds 5
+            $script:deployAllRtDeployment = Get-AzResource -ResourceGroupName $script:resourceGroup.ResourceGroupName -ResourceType 'Microsoft.Network/routeTables'  | Where-Object {$_.name -like "deployallrtwex*" -or $_.name -like "deployallrt2wex*"}
+            $script:deployAllRtDeployment.Count | Should -Be 3
+        }
+        #endregion
+
         #region Multiple deployments to test parallel deployment logic
         It "Deploy parallel storage accounts and compare to serial timing" {
             Set-PSFConfig -FullName AzOps.Core.AllowMultipleTemplateParameterFiles -Value $true
@@ -1232,7 +1252,7 @@ Describe "Repository" {
             )
             {Invoke-AzOpsPush -ChangeSet $changeSet} | Should -Not -Throw
             Start-Sleep -Seconds 30
-            $script:deployAllStaParamPathDeployment = Get-AzResource -ResourceGroupName $($script:resourceGroupParallelDeploy).ResourceGroupName -ResourceType 'Microsoft.Storage/storageAccounts'
+            $script:deployAllStaParamPathDeployment = Get-AzResource -ResourceGroupName $script:resourceGroupParallelDeploy.ResourceGroupName -ResourceType 'Microsoft.Storage/storageAccounts'
             $script:deployAllStaParamPathDeployment.Count | Should -Be 4
             $query = "resourcechanges | where resourceGroup =~ '$($($script:resourceGroupParallelDeploy).ResourceGroupName)' and properties.targetResourceType == 'microsoft.storage/storageaccounts' and properties.changeType == 'Create' | extend changeTime=todatetime(properties.changeAttributes.timestamp), targetResourceId=tostring(properties.targetResourceId) | summarize arg_max(changeTime, *) by targetResourceId | project changeTime, targetResourceId, properties.changeType, properties.targetResourceType | order by changeTime asc"
             $createTime = Search-AzGraph -Query $query -Subscription $script:subscriptionId

src/tests/templates/decoy.westeurope.bicep

@@ -0,0 +1,12 @@
+param name string
+param location string = resourceGroup().location
+
+resource symbolicname 'Microsoft.Network/routeTables@2023-04-01' = {
+  name: name
+  location: location
+  properties: {
+    disableBgpRoutePropagation: false
+    routes: [
+    ]
+  }
+}

src/tests/templates/decoy.westeurope.x123.parameters.json

@@ -0,0 +1,9 @@
+{
+  "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentParameters.json#",
+  "contentVersion": "1.0.0.0",
+  "parameters": {
+    "name": {
+      "value": "decoywex123"
+    }
+  }
+}

src/tests/templates/deployallrt.westeurope.bicep

@@ -0,0 +1,12 @@
+param name string
+param location string = resourceGroup().location
+
+resource symbolicname 'Microsoft.Network/routeTables@2023-04-01' = {
+  name: name
+  location: location
+  properties: {
+    disableBgpRoutePropagation: false
+    routes: [
+    ]
+  }
+}

src/tests/templates/deployallrt.westeurope.x123.parameters.json

@@ -0,0 +1,9 @@
+{
+  "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentParameters.json#",
+  "contentVersion": "1.0.0.0",
+  "parameters": {
+    "name": {
+      "value": "deployallrtwex123"
+    }
+  }
+}

src/tests/templates/deployallrt.westeurope.xabc.bicepparam

@@ -0,0 +1,3 @@
+using './deployallrt.westeurope.bicep'
+
+param name = toLower('deployallrtwexabc')

src/tests/templates/deployallrt2.westeurope.bicep

@@ -0,0 +1,12 @@
+param name string = 'deployallrt2wex123'
+param location string = resourceGroup().location
+
+resource symbolicname 'Microsoft.Network/routeTables@2023-04-01' = {
+  name: name
+  location: location
+  properties: {
+    disableBgpRoutePropagation: false
+    routes: [
+    ]
+  }
+}